Package org.languagetool.language.identifier

Class LanguageIdentifier

java.lang.Object

org.languagetool.language.identifier.LanguageIdentifier

Direct Known Subclasses:

DefaultLanguageIdentifier, SimpleLanguageIdentifier

public abstract class LanguageIdentifier
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
protected static class	LanguageIdentifier.ParsedLanguageLists

Field Summary

Fields

Modifier and Type	Field	Description
protected static final CommonWordsDetector	COMMON_WORDS_LANG_IDENTIFIER
protected static final int	CONSIDER_ONLY_PREFERRED_THRESHOLD
private static final Pattern	MAIL_REGEX
protected int	maxLength
private static final Pattern	MENTION
private static final Pattern	NBSP_INVIS_SEPARATOR
protected static final List<String>	NON_LATIN_CHARS_LANGUAGES
protected static final com.optimaize.langdetect.text.TextFilter	REMOVE_EMAIL_SIGNATURE_FILTER
protected static final com.optimaize.langdetect.text.TextFilter	REMOVE_MENTION_FILTER
protected static final com.optimaize.langdetect.text.TextFilter	REMOVE_NON_BREAKING_SPACES_FILTER
protected static final com.optimaize.langdetect.text.TextFilter	REMOVE_URL_FILTER
protected static final float	SCORE_THRESHOLD
private static final Pattern	SIGNATURE
protected static final UnicodeBasedDetector	UNICODE_BASED_LANG_IDENTIFIER
private static final Pattern	URL_REGEX

Constructor Summary

Constructors

Constructor	Description
LanguageIdentifier(int maxLength)

Method Summary

Modifier and Type	Method	Description
String	cleanAndShortenText(String text)
abstract Language	detectLanguage(String cleanText)
abstract DetectedLanguage	detectLanguage(String cleanText, List<String> noopLangsTmp, List<String> preferredLangsTmp)
abstract DetectedLanguage	detectLanguage(String cleanText, List<String> noopLangsTmp, List<String> preferredLangsTmp, boolean limitOnPreferredLangs)
abstract List<DetectedLanguage>	getDetectedLanguageScores(String cleanText, List<String> noopLangsTmp, List<String> preferredLangsTmp, boolean limitOnPreferredLangs, int count)
protected Map.Entry<String,Double>	getHighestScoringResult(Map<String,Double> probs)
protected Map<String,Double>	getOrderedScores(Map<String,Double> scores, int count)
protected LanguageIdentifier.ParsedLanguageLists	prepareDetectLanguage(String text, List<String> noopLangsTmp, List<String> preferredLangsTmp)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

URL_REGEX

private static final Pattern URL_REGEX

MAIL_REGEX

private static final Pattern MAIL_REGEX

SIGNATURE

private static final Pattern SIGNATURE

MENTION

private static final Pattern MENTION

NBSP_INVIS_SEPARATOR

private static final Pattern NBSP_INVIS_SEPARATOR

SCORE_THRESHOLD

protected static final float SCORE_THRESHOLD

See Also:

• Constant Field Values

CONSIDER_ONLY_PREFERRED_THRESHOLD

protected static final int CONSIDER_ONLY_PREFERRED_THRESHOLD

See Also:

• Constant Field Values

NON_LATIN_CHARS_LANGUAGES

protected static final List<String> NON_LATIN_CHARS_LANGUAGES

REMOVE_EMAIL_SIGNATURE_FILTER

protected static final com.optimaize.langdetect.text.TextFilter REMOVE_EMAIL_SIGNATURE_FILTER

REMOVE_MENTION_FILTER

protected static final com.optimaize.langdetect.text.TextFilter REMOVE_MENTION_FILTER

REMOVE_NON_BREAKING_SPACES_FILTER

protected static final com.optimaize.langdetect.text.TextFilter REMOVE_NON_BREAKING_SPACES_FILTER

REMOVE_URL_FILTER

protected static final com.optimaize.langdetect.text.TextFilter REMOVE_URL_FILTER

UNICODE_BASED_LANG_IDENTIFIER

protected static final UnicodeBasedDetector UNICODE_BASED_LANG_IDENTIFIER

COMMON_WORDS_LANG_IDENTIFIER

protected static final CommonWordsDetector COMMON_WORDS_LANG_IDENTIFIER

maxLength

protected int maxLength

Constructor Details

LanguageIdentifier

public LanguageIdentifier(int maxLength)

Method Details

detectLanguage

@Nullable
public abstract DetectedLanguage detectLanguage(String cleanText,
 List<String> noopLangsTmp,
 List<String> preferredLangsTmp)

Parameters:

cleanText - a cleanText as returned by cleanAndShortenText(String)

noopLangsTmp - list of codes that are detected but will lead to the NoopLanguage that has no rules

Returns:

language or null if language could not be identified

Since:

4.4 (new parameter noopLangs, changed return type to DetectedLanguage)

detectLanguage

@Nullable
public abstract DetectedLanguage detectLanguage(String cleanText,
 List<String> noopLangsTmp,
 List<String> preferredLangsTmp,
 boolean limitOnPreferredLangs)

getDetectedLanguageScores

@NotNull
public abstract List<DetectedLanguage> getDetectedLanguageScores(String cleanText,
 List<String> noopLangsTmp,
 List<String> preferredLangsTmp,
 boolean limitOnPreferredLangs,
 int count)

detectLanguage

@Nullable
public abstract Language detectLanguage(String cleanText)

Parameters:

cleanText - a cleanText as returned by cleanAndShortenText(String)

Returns:

language or null if language could not be identified

Since:

4.4 (new parameter noopLangs, changed return type to DetectedLanguage)

cleanAndShortenText

public String cleanAndShortenText(String text)

Since:

5.8

prepareDetectLanguage

@Nullable
protected LanguageIdentifier.ParsedLanguageLists prepareDetectLanguage(String text,
 List<String> noopLangsTmp,
 List<String> preferredLangsTmp)

getHighestScoringResult

protected Map.Entry<String,Double> getHighestScoringResult(Map<String,Double> probs)

getOrderedScores

protected Map<String,Double> getOrderedScores(Map<String,Double> scores,
 int count)