chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:37:18 +08:00
commit 1fad2ccf05
698 changed files with 81751 additions and 0 deletions
@@ -0,0 +1,28 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2020-12-27 12:07 AM</create-date>
*
* <copyright file="Input.java">
* Copyright (c) 2020, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful;
/**
* @author hankcs
*/
public class BaseInput
{
public String[] tasks;
public String[] skip_tasks;
public String language;
public BaseInput(String[] tasks, String[] skipTasks, String language)
{
this.tasks = tasks;
this.skip_tasks = skipTasks;
this.language = language;
}
}
@@ -0,0 +1,32 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2021-10-16 4:43 PM</create-date>
*
* <copyright file="CoreferenceResolutionOutput.java">
* Copyright (c) 2021, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* A data class for coreference resolution
*
* @author hankcs
*/
public class CoreferenceResolutionOutput
{
public List<Set<Span>> clusters;
public ArrayList<String> tokens;
public CoreferenceResolutionOutput(List<Set<Span>> clusters, ArrayList<String> tokens)
{
this.clusters = clusters;
this.tokens = tokens;
}
}
@@ -0,0 +1,25 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2020-12-27 12:09 AM</create-date>
*
* <copyright file="TextInput.java">
* Copyright (c) 2020, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful;
/**
* @author hankcs
*/
public class DocumentInput extends BaseInput
{
public String text;
public DocumentInput(String text, String[] tasks, String[] skipTasks, String language)
{
super(tasks, skipTasks, language);
this.text = text;
}
}
@@ -0,0 +1,629 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2020-12-26 11:54 PM</create-date>
*
* <copyright file="HanLPClient.java">
* Copyright (c) 2020, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.hankcs.hanlp.restful.mrp.MeaningRepresentation;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.*;
/**
* A RESTful client implementing the data format specification of HanLP.
*
* @author hankcs
* @see <a href="https://hanlp.hankcs.com/docs/data_format.html">Data Format</a>
*/
public class HanLPClient
{
private String url;
private String auth;
private String language;
private int timeout;
private ObjectMapper mapper;
/**
* @param url An API endpoint to a service provider.
* @param auth An auth key licenced by a service provider.
* @param language The language this client will be expecting. Contact the service provider for the list of
* languages supported. Conventionally, zh is used for Chinese and mul for multilingual.
* Leave null to use the default language on server.
* @param timeout Maximum waiting time in seconds for a request.
*/
public HanLPClient(String url, String auth, String language, int timeout)
{
if (auth == null)
{
auth = System.getenv().getOrDefault("HANLP_AUTH", null);
}
this.url = url;
this.auth = auth;
this.language = language;
this.timeout = timeout * 1000;
this.mapper = new ObjectMapper();
}
/**
* @param url An API endpoint to a service provider.
* @param auth An auth key licenced by a service provider.
*/
public HanLPClient(String url, String auth)
{
this(url, auth, null, 5);
}
/**
* Parse a raw document.
*
* @param text Document content which can have multiple sentences.
* @param tasks Tasks to perform.
* @param skipTasks Tasks to skip.
* @return Parsed annotations.
* @throws IOException HTTP exception.
* @see <a href="https://hanlp.hankcs.com/docs/data_format.html">Data Format</a>
*/
public Map<String, List> parse(String text, String[] tasks, String[] skipTasks) throws IOException
{
//noinspection unchecked
return mapper.readValue(post("/parse", new DocumentInput(text, tasks, skipTasks, language)), Map.class);
}
/**
* Parse a raw document.
*
* @param text Document content which can have multiple sentences.
* @return Parsed annotations.
* @throws IOException HTTP exception.
* @see <a href="https://hanlp.hankcs.com/docs/data_format.html">Data Format</a>
*/
public Map<String, List> parse(String text) throws IOException
{
return parse(text, null, null);
}
/**
* Parse an array of sentences.
*
* @param sentences Multiple sentences to parse.
* @param tasks Tasks to perform.
* @param skipTasks Tasks to skip.
* @return Parsed annotations.
* @throws IOException HTTP exception.
* @see <a href="https://hanlp.hankcs.com/docs/data_format.html">Data Format</a>
*/
public Map<String, List> parse(String[] sentences, String[] tasks, String[] skipTasks) throws IOException
{
//noinspection unchecked
return mapper.readValue(post("/parse", new SentenceInput(sentences, tasks, skipTasks, language)), Map.class);
}
/**
* Parse an array of sentences.
*
* @param sentences Multiple sentences to parse.
* @return Parsed annotations.
* @throws IOException HTTP exception.
* @see <a href="https://hanlp.hankcs.com/docs/data_format.html">Data Format</a>
*/
public Map<String, List> parse(String[] sentences) throws IOException
{
return parse(sentences, null, null);
}
/**
* Parse an array of pre-tokenized sentences.
*
* @param tokens Multiple pre-tokenized sentences to parse.
* @param tasks Tasks to perform.
* @param skipTasks Tasks to skip.
* @return Parsed annotations.
* @throws IOException HTTP exception.
* @see <a href="https://hanlp.hankcs.com/docs/data_format.html">Data Format</a>
*/
public Map<String, List> parse(String[][] tokens, String[] tasks, String[] skipTasks) throws IOException
{
//noinspection unchecked
return mapper.readValue(post("/parse", new TokenInput(tokens, tasks, skipTasks, language)), Map.class);
}
/**
* Parse an array of pre-tokenized sentences.
*
* @param tokens Multiple pre-tokenized sentences to parse.
* @return Parsed annotations.
* @throws IOException HTTP exception.
* @see <a href="https://hanlp.hankcs.com/docs/data_format.html">Data Format</a>
*/
public Map<String, List> parse(String[][] tokens) throws IOException
{
return parse(tokens, null, null);
}
/**
* Split a document into sentences and tokenize them.
*
* @param text A document.
* @param coarse Whether to perform coarse-grained or fine-grained tokenization.
* @return A list of tokenized sentences.
* @throws IOException HTTP exception.
*/
public List<List<String>> tokenize(String text, Boolean coarse) throws IOException
{
String[] tasks;
if (coarse != null)
{
if (coarse)
tasks = new String[]{"tok/coarse"};
else
tasks = new String[]{"tok/fine"};
}
else
tasks = new String[]{"tok"};
Map<String, List> doc = parse(text, tasks, null);
//noinspection unchecked
return doc.values().iterator().next();
}
/**
* Split a document into sentences and tokenize them using fine-grained standard.
*
* @param text A document.
* @return A list of tokenized sentences.
* @throws IOException HTTP exception.
*/
public List<List<String>> tokenize(String text) throws IOException
{
return tokenize(text, null);
}
/**
* Text style transfer aims to change the style of the input text to the target style while preserving its content.
*
* @param text Source text.
* @param targetStyle Target style.
* @return Text of the target style.
*/
public List<String> textStyleTransfer(List<String> text, String targetStyle) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("target_style", targetStyle);
input.put("language", language);
//noinspection unchecked
return mapper.readValue(post("/text_style_transfer", input), List.class);
}
/**
* Text style transfer aims to change the style of the input text to the target style while preserving its content.
*
* @param text Source text.
* @param targetStyle Target style.
* @return Text of the target style.
*/
public String textStyleTransfer(String text, String targetStyle) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("target_style", targetStyle);
input.put("language", language);
return mapper.readValue(post("/text_style_transfer", input), String.class);
}
/**
* Grammatical Error Correction (GEC) is the task of correcting different kinds of errors in text such as
* spelling, punctuation, grammatical, and word choice errors.
*
* @param text Text potentially containing different kinds of errors such as spelling, punctuation,
* grammatical, and word choice errors.
* @return Corrected text.
*/
public List<String> grammaticalErrorCorrection(List<String> text) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("language", language);
//noinspection unchecked
return mapper.readValue(post("/grammatical_error_correction", input), List.class);
}
/**
* Grammatical Error Correction (GEC) is the task of correcting different kinds of errors in text such as
* spelling, punctuation, grammatical, and word choice errors.
*
* @param text Text potentially containing different kinds of errors such as spelling, punctuation,
* grammatical, and word choice errors.
* @return Corrected text.
*/
public String[] grammaticalErrorCorrection(String[] text) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("language", language);
//noinspection unchecked
return mapper.readValue(post("/grammatical_error_correction", input), String[].class);
}
/**
* Grammatical Error Correction (GEC) is the task of correcting different kinds of errors in text such as
* spelling, punctuation, grammatical, and word choice errors.
*
* @param text Text potentially containing different kinds of errors such as spelling, punctuation,
* grammatical, and word choice errors.
* @return Corrected text.
*/
public String grammaticalErrorCorrection(String text) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("language", language);
return mapper.readValue(post("/grammatical_error_correction", input), String.class);
}
/**
* Semantic textual similarity deals with determining how similar two pieces of texts are.
*
* @param textA The first text.
* @param textB The second text.
* @return Their similarity.
* @throws IOException HTTP errors.
*/
public Double semanticTextualSimilarity(String textA, String textB) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", new String[]{textA, textB});
input.put("language", language);
return mapper.readValue(post("/semantic_textual_similarity", input), Double.class);
}
/**
* Semantic textual similarity deals with determining how similar two pieces of texts are.
*
* @param text The pairs of text.
* @return Their similarities.
* @throws IOException HTTP errors.
*/
public List<Double> semanticTextualSimilarity(String[][] text) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("language", language);
//noinspection unchecked
return mapper.readValue(post("/semantic_textual_similarity", input), List.class);
}
/**
* Coreference resolution is the task of clustering mentions in text that refer to the same underlying real world entities.
*
* @param text A piece of text, usually a document without tokenization.
* @return Coreference resolution clusters and tokens.
* @throws IOException HTTP errors.
*/
public CoreferenceResolutionOutput coreferenceResolution(String text) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("language", language);
//noinspection unchecked
Map<String, List> response = mapper.readValue(post("/coreference_resolution", input), Map.class);
//noinspection unchecked
List<List<List>> clusters = response.get("clusters");
return new CoreferenceResolutionOutput(_convert_clusters(clusters), (ArrayList<String>) response.get("tokens"));
}
/**
* Coreference resolution is the task of clustering mentions in text that refer to the same underlying real world entities.
*
* @param tokens A list of sentences where each sentence is a list of tokens.
* @param speakers A list of speakers where each speaker is a String representing the speaker's ID, e.g., "Tom".
* @return Coreference resolution clusters.
* @throws IOException HTTP errors.
*/
public List<Set<Span>> coreferenceResolution(String[][] tokens, String[] speakers) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("tokens", tokens);
input.put("speakers", speakers);
input.put("language", language);
//noinspection unchecked
List<List<List>> clusters = mapper.readValue(post("/coreference_resolution", input), List.class);
return _convert_clusters(clusters);
}
/**
* Coreference resolution is the task of clustering mentions in text that refer to the same underlying real world entities.
*
* @param tokens A list of sentences where each sentence is a list of tokens.
* @return Coreference resolution clusters.
* @throws IOException HTTP errors.
*/
public List<Set<Span>> coreferenceResolution(String[][] tokens) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("tokens", tokens);
input.put("language", language);
//noinspection unchecked
List<List<List>> clusters = mapper.readValue(post("/coreference_resolution", input), List.class);
return _convert_clusters(clusters);
}
private static List<Set<Span>> _convert_clusters(List<List<List>> clusters)
{
List<Set<Span>> results = new ArrayList<>(clusters.size());
for (List<List> cluster : clusters)
{
Set<Span> spans = new LinkedHashSet<>();
for (List span : cluster)
{
spans.add(new Span((String) span.get(0), (Integer) span.get(1), (Integer) span.get(2)));
}
results.add(spans);
}
return results;
}
/**
* Abstract Meaning Representation (AMR) captures “who is doing what to whom” in a sentence. Each sentence is
* represented as a rooted, directed, acyclic graph consisting of nodes (concepts) and edges (relations).
*
* @param text A piece of text, usually a document without tokenization.
* @return AMR graphs.
* @throws IOException HTTP errors.
*/
public MeaningRepresentation[] abstractMeaningRepresentation(String text) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("language", language);
return mapper.readValue(post("/abstract_meaning_representation", input), MeaningRepresentation[].class);
}
/**
* Abstract Meaning Representation (AMR) captures “who is doing what to whom” in a sentence. Each sentence is
* represented as a rooted, directed, acyclic graph consisting of nodes (concepts) and edges (relations).
*
* @param tokens A list of sentences where each sentence is a list of tokens.
* @return AMR graphs.
* @throws IOException HTTP errors.
*/
public MeaningRepresentation[] abstractMeaningRepresentation(String[][] tokens) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("tokens", tokens);
input.put("language", language);
return mapper.readValue(post("/abstract_meaning_representation", input), MeaningRepresentation[].class);
}
/**
* Keyphrase extraction aims to identify keywords or phrases reflecting the main topics of a document.
*
* @param text The text content of the document. Preferably the concatenation of the title and the content.
* @param topk The number of top-K ranked keywords or keyphrases.
* @return A dictionary containing each keyphrase and its ranking score s between 0 and 1.
* @throws IOException HTTP errors.
*/
public Map<String, Double> keyphraseExtraction(String text, int topk) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("topk", topk);
input.put("language", language);
//noinspection unchecked
return mapper.readValue(post("/keyphrase_extraction", input), LinkedHashMap.class);
}
/**
* Single document summarization is the task of selecting a subset of the sentences which best
* represents a summary of the document, with a balance of salience and redundancy.
*
* @param text The text content of the document.
* @return A dictionary containing each sentence and its ranking score s between 0 and 1.
* @throws IOException HTTP errors.
*/
public Map<String, Double> extractiveSummarization(String text) throws IOException
{
return extractiveSummarization(text, 3);
}
/**
* Single document summarization is the task of selecting a subset of the sentences which best
* represents a summary of the document, with a balance of salience and redundancy.
*
* @param text The text content of the document.
* @param topk The maximum number of top-K ranked sentences. Note that due to Trigram Blocking tricks, the actual
* number of returned sentences could be less than ``topk``.
* @return A dictionary containing each sentence and its ranking score s between 0 and 1.
* @throws IOException HTTP errors.
*/
public Map<String, Double> extractiveSummarization(String text, int topk) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("topk", topk);
input.put("language", language);
//noinspection unchecked
return mapper.readValue(post("/extractive_summarization", input), LinkedHashMap.class);
}
/**
* Abstractive Summarization is the task of generating a short and concise summary that captures the
* salient ideas of the source text. The generated summaries potentially contain new phrases and sentences that
* may not appear in the source text.
*
* @param text The text content of the document.
* @return Summarization.
* @throws IOException HTTP errors.
*/
public String abstractiveSummarization(String text) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("language", language);
//noinspection unchecked
return mapper.readValue(post("/abstractive_summarization", input), String.class);
}
/**
* Text classification is the task of assigning a sentence or document an appropriate category.
* The categories depend on the chosen dataset and can range from topics.
*
* @param text The text content of the document.
* @param model The model to use for prediction.
* @return Classification results.
* @throws IOException HTTP errors.
*/
public String textClassification(String text, String model) throws IOException
{
return (String) textClassification(text, model, false, false);
}
/**
* Sentiment analysis is the task of classifying the polarity of a given text. For instance,
* a text-based tweet can be categorized into either "positive", "negative", or "neutral".
*
* @param text The text content of the document.
* @return Sentiment polarity as a numerical value which measures how positive the sentiment is.
* @throws IOException HTTP errors.
*/
public Double sentimentAnalysis(String text) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("language", language);
//noinspection unchecked
return mapper.readValue(post("/sentiment_analysis", input), Double.class);
}
/**
* Text classification is the task of assigning a sentence or document an appropriate category.
* The categories depend on the chosen dataset and can range from topics.
*
* @param text A document or a list of documents.
* @param model The model to use for prediction.
* @param topk `true` or `int` to return the top-k languages.
* @param prob Return also probabilities.
* @return Classification results.
* @throws IOException HTTP errors.
*/
public Object textClassification(Object text, String model, Object topk, boolean prob) throws IOException
{
Map<String, Object> input = new HashMap<>();
input.put("text", text);
input.put("model", model);
input.put("topk", topk);
input.put("prob", prob);
//noinspection unchecked
return mapper.readValue(post("/text_classification", input), Object.class);
}
/**
* Recognize the language of a given text.
*
* @param text The text content of the document.
* @return Identified language in <a href="https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">ISO 639-1 codes</a>.
* @throws IOException HTTP errors.
*/
public String languageIdentification(String text) throws IOException
{
return textClassification(text, "lid");
}
/**
* Recognize the language of a given text.
*
* @param text The text content of the document.
* @return Identified language in <a href="https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">ISO 639-1 codes</a>.
* @throws IOException HTTP errors.
*/
public List<String> languageIdentification(String[] text) throws IOException
{
return (List<String>) textClassification(text, "lid", false, false);
}
/**
* Keyphrase extraction aims to identify keywords or phrases reflecting the main topics of a document.
*
* @param text The text content of the document. Preferably the concatenation of the title and the content.
* @return A dictionary containing 10 keyphrases and their ranking scores s between 0 and 1.
* @throws IOException HTTP errors.
*/
public Map<String, Double> keyphraseExtraction(String text) throws IOException
{
return keyphraseExtraction(text, 10);
}
private String post(String api, Object input_) throws IOException
{
URL url = new URL(this.url + api);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setRequestMethod("POST");
if (auth != null)
con.setRequestProperty("Authorization", "Basic " + auth);
con.setRequestProperty("Content-Type", "application/json; utf-8");
con.setRequestProperty("Accept", "application/json");
con.setDoOutput(true);
con.setConnectTimeout(timeout);
con.setReadTimeout(timeout);
String jsonInputString = mapper.writeValueAsString(input_);
try (OutputStream os = con.getOutputStream())
{
byte[] input = jsonInputString.getBytes(StandardCharsets.UTF_8);
os.write(input, 0, input.length);
}
int code = con.getResponseCode();
if (code != 200)
{
StringBuilder response = new StringBuilder();
try (BufferedReader br = new BufferedReader(new InputStreamReader(con.getErrorStream(), StandardCharsets.UTF_8)))
{
String responseLine;
while ((responseLine = br.readLine()) != null)
{
response.append(responseLine.trim());
}
}
String error = String.format("Request failed, status code = %d, error = %s", code, con.getResponseMessage());
try
{
Map detail = mapper.readValue(response.toString(), Map.class);
error = (String) detail.get("detail");
}
catch (Exception ignored)
{
}
throw new IOException(error);
}
StringBuilder response = new StringBuilder();
try (BufferedReader br = new BufferedReader(new InputStreamReader(con.getInputStream(), StandardCharsets.UTF_8)))
{
String responseLine;
while ((responseLine = br.readLine()) != null)
{
response.append(responseLine.trim());
}
}
return response.toString();
}
}
@@ -0,0 +1,25 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2020-12-27 12:09 AM</create-date>
*
* <copyright file="SentenceInput.java">
* Copyright (c) 2020, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful;
/**
* @author hankcs
*/
public class SentenceInput extends BaseInput
{
public String[] text;
public SentenceInput(String[] text, String[] tasks, String[] skipTasks, String language)
{
super(tasks, skipTasks, language);
this.text = text;
}
}
@@ -0,0 +1,64 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2021-10-16 4:26 PM</create-date>
*
* <copyright file="Span.java">
* Copyright (c) 2021, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful;
import java.util.Objects;
/**
* A common data format to represent a span.
*
* @author hankcs
*/
public class Span
{
/**
* The raw form of a span, which can be either a token, an entity or a mention etc.
*/
public String form;
/**
* The inclusive beginning offset of a span.
*/
public int begin;
/**
* The exclusive ending offset of a span.
*/
public int end;
public Span(String form, int begin, int end)
{
this.form = form;
this.begin = begin;
this.end = end;
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Span span = (Span) o;
return begin == span.begin &&
end == span.end &&
form.equals(span.form);
}
@Override
public int hashCode()
{
return Objects.hash(form, begin, end);
}
@Override
public String toString()
{
return String.format("[%d, %d) = %s", begin, end, form);
}
}
@@ -0,0 +1,25 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2020-12-27 12:09 AM</create-date>
*
* <copyright file="TokenInput.java">
* Copyright (c) 2020, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful;
/**
* @author hankcs
*/
public class TokenInput extends BaseInput
{
public String[][] tokens;
public TokenInput(String[][] tokens, String[] tasks, String[] skipTasks, String language)
{
super(tasks, skipTasks, language);
this.tokens = tokens;
}
}
@@ -0,0 +1,20 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2022-04-13 8:58 AM</create-date>
*
* <copyright file="Anchor.java">
* Copyright (c) 2022, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful.mrp;
/**
* @author hankcs
*/
public class Anchor
{
public String from;
public String to;
}
@@ -0,0 +1,21 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2022-04-13 9:01 AM</create-date>
*
* <copyright file="Edge.java">
* Copyright (c) 2022, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful.mrp;
/**
* @author hankcs
*/
public class Edge
{
public int source;
public int target;
public String label;
}
@@ -0,0 +1,26 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2022-04-13 8:57 AM</create-date>
*
* <copyright file="MeaningRepresentation.java">
* Copyright (c) 2022, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful.mrp;
/**
* Graph-based meaning representation.
*
* @author hankcs
*/
public class MeaningRepresentation
{
public String id;
public String input;
public Node[] nodes;
public Edge[] edges;
public String[] tops;
public String framework;
}
@@ -0,0 +1,23 @@
/*
* <author>Han He</author>
* <email>me@hankcs.com</email>
* <create-date>2022-04-13 8:57 AM</create-date>
*
* <copyright file="Node.java">
* Copyright (c) 2022, Han He. All Rights Reserved, http://www.hankcs.com/
* See LICENSE file in the project root for full license information.
* </copyright>
*/
package com.hankcs.hanlp.restful.mrp;
/**
* @author hankcs
*/
public class Node
{
public int id;
public String label;
public String[] properties;
public String[] values;
public Anchor[] anchors;
}
@@ -0,0 +1,192 @@
package com.hankcs.hanlp.restful;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
class HanLPClientTest
{
HanLPClient client;
@BeforeEach
void setUp()
{
client = new HanLPClient("https://hanlp.hankcs.com/api", null);
}
@org.junit.jupiter.api.Test
void parseText() throws IOException
{
Map<String, List> doc = client.parse("2021年HanLPv2.1为生产环境带来次世代最先进的多语种NLP技术。英首相与特朗普通电话讨论华为与苹果公司。");
prettyPrint(doc);
}
@org.junit.jupiter.api.Test
void parseSentences() throws IOException
{
Map<String, List> doc = client.parse(new String[]{
"2021年HanLPv2.1为生产环境带来次世代最先进的多语种NLP技术。",
"英首相与特朗普通电话讨论华为与苹果公司。"
});
prettyPrint(doc);
}
@org.junit.jupiter.api.Test
void parseTokens() throws IOException
{
Map<String, List> doc = client.parse(new String[][]{
new String[]{"2021年", "HanLPv2.1", "", "生产", "环境", "带来", "", "世代", "", "先进", "", "多语种", "NLP", "技术", ""},
new String[]{"", "首相", "", "特朗普", "", "电话", "讨论", "华为", "", "苹果", "公司", ""},
});
prettyPrint(doc);
}
@Test
void parseCoarse() throws IOException
{
Map<String, List> doc = client.parse(
"阿婆主来到北京立方庭参观自然语义科技公司。",
new String[]{"tok/coarse", "pos", "dep"},
new String[]{"tok/fine"});
prettyPrint(doc);
}
@Test
void tokenize() throws IOException
{
List<List<String>> fine = client.tokenize("2021年HanLPv2.1为生产环境带来次世代最先进的多语种NLP技术。阿婆主来到北京立方庭参观自然语义科技公司。");
System.out.println(fine);
List<List<String>> coarse = client.tokenize("2021年HanLPv2.1为生产环境带来次世代最先进的多语种NLP技术。阿婆主来到北京立方庭参观自然语义科技公司。", true);
System.out.println(coarse);
}
@Test
void textStyleTransfer() throws IOException
{
String doc = client.textStyleTransfer("国家对中石油抱有很大的期望.", "gov_doc");
prettyPrint(doc);
}
@Test
void semanticTextualSimilarity() throws IOException
{
Double similarity = client.semanticTextualSimilarity("看图猜一电影名", "看图猜电影");
prettyPrint(similarity);
List<Double> similarities = client.semanticTextualSimilarity(new String[][]{
new String[]{"看图猜一电影名", "看图猜电影"},
new String[]{"北京到上海的动车票", "上海到北京的动车票"}
});
for (Double similarityPerPair : similarities)
{
prettyPrint(similarityPerPair);
}
}
@Test
void coreferenceResolutionText() throws IOException
{
CoreferenceResolutionOutput clusters = client.coreferenceResolution("我姐送我她的猫。我很喜欢它。");
prettyPrint(clusters);
}
@Test
void coreferenceResolutionTokens() throws IOException
{
List<Set<Span>> clusters = client.coreferenceResolution(
new String[][]{
new String[]{"", "", "", "", "", "", "", ""},
new String[]{"", "", "喜欢", "", ""}});
prettyPrint(clusters);
}
@Test
void coreferenceResolutionTokensWithSpeakers() throws IOException
{
List<Set<Span>> clusters = client.coreferenceResolution(
new String[][]{
new String[]{"", "", "", "", "", "", "", ""},
new String[]{"", "", "喜欢", "", ""}},
new String[]{"张三", "张三"});
prettyPrint(clusters);
}
@Test
void keyphraseExtraction() throws IOException
{
prettyPrint(client.keyphraseExtraction(
"自然语言处理是一门博大精深的学科,掌握理论才能发挥出HanLP的全部性能。" +
"《自然语言处理入门》是一本配套HanLP的NLP入门书,助你零起点上手自然语言处理。", 3));
}
@Test
void extractiveSummarization() throws IOException
{
prettyPrint(client.extractiveSummarization(
"据DigiTimes报道,在上海疫情趋缓,防疫管控开始放松后,苹果供应商广达正在逐步恢复其中国工厂的MacBook产品生产。\n" +
"据供应链消息人士称,生产厂的订单拉动情况正在慢慢转强,这会提高MacBook Pro机型的供应量,并缩短苹果客户在过去几周所经历的延长交货时间。\n" +
"仍有许多苹果笔记本用户在等待3月和4月订购的MacBook Pro机型到货,由于苹果的供应问题,他们的发货时间被大大推迟了。\n" +
"据分析师郭明錤表示,广达是高端MacBook Pro的唯一供应商,自防疫封控依赖,MacBook Pro大部分型号交货时间增加了三到五周,\n" +
"一些高端定制型号的MacBook Pro配置要到6月底到7月初才能交货。\n" +
"尽管MacBook Pro的生产逐渐恢复,但供应问题预计依然影响2022年第三季度的产品销售。\n" +
"苹果上周表示,防疫措施和元部件短缺将继续使其难以生产足够的产品来满足消费者的强劲需求,这最终将影响苹果6月份的收入。"));
}
@Test
void abstractiveSummarization() throws IOException
{
prettyPrint(client.abstractiveSummarization(
"每经AI快讯,2月4日,长江证券研究所金属行业首席分析师王鹤涛表示,2023年海外经济衰退,美债现处于历史高位,\n" +
"黄金的趋势是值得关注的;在国内需求修复的过程中,看好大金属品种中的铜铝钢。\n" +
"此外,在细分的小品种里,建议关注两条主线,一是新能源,比如锂、钴、镍、稀土,二是专精特新主线。(央视财经)"));
}
@Test
void abstractMeaningRepresentationText() throws IOException
{
prettyPrint(client.abstractMeaningRepresentation("男孩希望女孩相信他。阿婆主来到北京立方庭参观自然语义科技公司。"));
}
@Test
void abstractMeaningRepresentationTokens() throws IOException
{
prettyPrint(client.abstractMeaningRepresentation(new String[][]{
new String[]{"2021年", "HanLPv2.1", "", "生产", "环境", "带来", "", "世代", "", "先进", "", "多语种", "NLP", "技术", ""},
new String[]{"", "首相", "", "特朗普", "", "电话", "讨论", "华为", "", "苹果", "公司", ""}}));
}
@Test
void grammaticalErrorCorrection() throws IOException
{
prettyPrint(client.grammaticalErrorCorrection(new String[]{"每个青年都应当有远大的报复。", "有的同学对语言很兴趣。"}));
}
@Test
void languageIdentification() throws IOException
{
prettyPrint(client.languageIdentification(new String[]{
"In 2021, HanLPv2.1 delivers state-of-the-art multilingual NLP techniques to production environment.",
"2021年、HanLPv2.1は次世代の最先端多言語NLP技術を本番環境に導入します。",
"2021年 HanLPv2.1为生产环境带来次世代最先进的多语种NLP技术。",
}));
}
@Test
void sentimentAnalysis() throws IOException
{
prettyPrint(client.sentimentAnalysis(
"“这是一部男人必看的电影。”人人都这么说。但单纯从性别区分,就会让这电影变狭隘。《肖申克的救赎》突破了男人电影的局限,通篇几乎充满令人难以置信的温馨基调,而电影里最伟大的主题是“希望”。 当我们无奈地遇到了如同肖申克一般囚禁了心灵自由的那种囹圄,我们是无奈的老布鲁克,灰心的瑞德,还是智慧的安迪?运用智慧,信任希望,并且勇敢面对恐惧心理,去打败它? 经典的电影之所以经典,因为他们都在做同一件事——让你从不同的角度来欣赏希望的美好。"
));
}
void prettyPrint(Object object) throws JsonProcessingException
{
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(object));
}
}
@@ -0,0 +1,33 @@
package com.hankcs.hanlp.restful;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.hankcs.hanlp.restful.mrp.MeaningRepresentation;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
class MeaningRepresentationTest
{
@Test
void parseText() throws IOException
{
String json = "[{\"id\": \"0\", \"input\": \"北京 大学 计算 语言学 研究所 和 富士通 研究 开发 中心 有限公司 , 得到 了 人民日报社 新闻 信息 中心 的 语料库 。\", \"nodes\": [{\"id\": 0, \"label\": \"name\", \"properties\": [\"op1\", \"op2\"], \"values\": [\"北京\", \"大学\"], \"anchors\": [{\"from\": 0, \"to\": 2}, {\"from\": 3, \"to\": 5}]}, {\"id\": 1, \"label\": \"university\", \"anchors\": []}, {\"id\": 2, \"label\": \"name\", \"properties\": [\"op1\", \"op2\", \"op4\"], \"values\": [\"计算\", \"语言学\", \"\"], \"anchors\": [{\"from\": 6, \"to\": 8}, {\"from\": 9, \"to\": 12}, {\"from\": 13, \"to\": 16}]}, {\"id\": 3, \"label\": \"research-institute\", \"anchors\": []}, {\"id\": 4, \"label\": \"and\", \"anchors\": []}, {\"id\": 5, \"label\": \"name\", \"properties\": [\"op1\", \"op2\", \"op3\", \"op4\", \"op5\"], \"values\": [\"富士通\", \"研究\", \"开发\", \"中心\", \"有限公司\"], \"anchors\": [{\"from\": 19, \"to\": 22}, {\"from\": 23, \"to\": 25}, {\"from\": 26, \"to\": 28}, {\"from\": 29, \"to\": 31}, {\"from\": 32, \"to\": 36}]}, {\"id\": 6, \"label\": \"company\", \"anchors\": []}, {\"id\": 7, \"label\": \"得到-01\", \"anchors\": [{\"from\": 39, \"to\": 41}]}, {\"id\": 8, \"label\": \"\", \"anchors\": [{\"from\": 42, \"to\": 43}]}, {\"id\": 9, \"label\": \"name\", \"properties\": [\"op1\"], \"values\": [\"人民日报社\"], \"anchors\": [{\"from\": 44, \"to\": 49}]}, {\"id\": 10, \"label\": \"organization\", \"anchors\": []}, {\"id\": 11, \"label\": \"name\", \"properties\": [\"op1\", \"op2\", \"op3\"], \"values\": [\"新闻\", \"信息\", \"中心\"], \"anchors\": [{\"from\": 50, \"to\": 52}, {\"from\": 53, \"to\": 55}, {\"from\": 56, \"to\": 58}]}, {\"id\": 12, \"label\": \"organization\", \"anchors\": []}, {\"id\": 13, \"label\": \"语料库\", \"anchors\": [{\"from\": 61, \"to\": 64}]}], \"edges\": [{\"source\": 7, \"target\": 8, \"label\": \"aspect\"}, {\"source\": 7, \"target\": 4, \"label\": \"arg0\"}, {\"source\": 10, \"target\": 9, \"label\": \"name\"}, {\"source\": 4, \"target\": 6, \"label\": \"op2\"}, {\"source\": 7, \"target\": 13, \"label\": \"arg1\"}, {\"source\": 6, \"target\": 5, \"label\": \"name\"}, {\"source\": 12, \"target\": 11, \"label\": \"name\"}, {\"source\": 3, \"target\": 2, \"label\": \"name\"}, {\"source\": 1, \"target\": 0, \"label\": \"name\"}, {\"source\": 13, \"target\": 12, \"label\": \"poss\"}, {\"source\": 4, \"target\": 3, \"label\": \"op1\"}, {\"source\": 12, \"target\": 9, \"label\": \"name\"}, {\"source\": 1, \"target\": 3, \"label\": \"part\"}], \"tops\": [7], \"framework\": \"amr\"}]";
ObjectMapper mapper = new ObjectMapper();
MeaningRepresentation[] graphs = mapper.readValue(json, MeaningRepresentation[].class);
prettyPrint(graphs);
}
void prettyPrint(Object object) throws JsonProcessingException
{
ObjectMapper mapper = new ObjectMapper();
System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(object));
}
}