// Copyright (c) 2009 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Baybayin (ancient script of the Philippines) is detected as TAGALOG.
// Chu Nom (Vietnamese ancient Han characters) is detected as VIETNAMESE.
// HAITIAN_CREOLE is detected as such.
// NORWEGIAN and NORWEGIAN_N are detected separately (but not robustly)
// PORTUGUESE, PORTUGUESE_P, and PORTUGUESE_B are all detected as PORTUGUESE.
// ROMANIAN-Latin is detected as ROMANIAN; ROMANIAN-Cyrillic as MOLDAVIAN.
// SERBO_CROATIAN, BOSNIAN, CROATIAN, SERBIAN, MONTENEGRIN in the Latin script
//  are all detected as CROATIAN; in the Cyrillic script as SERBIAN.
// Zhuang is detected in the Latin script only.
//
// The Google interface languages X_PIG_LATIN and X_KLINGON are detected in the
//  extended calls ExtDetectLanguageSummary(). BorkBorkBork, ElmerFudd, and
//  Hacker are not detected (too little training data).
//
// UNKNOWN_LANGUAGE is returned if no language's internal reliablity measure
//  is high enough. This happens with non-text input such as the bytes of a
//  JPEG, and also with some text in languages outside the Google Language
//  enum, such as Ilonggo.
//
// The following languages are detected in multiple scripts:
//  AZERBAIJANI (Latin, Cyrillic*, Arabic*)
//  BURMESE (Latin, Myanmar)
//  HAUSA (Latin, Arabic)
//  KASHMIRI (Arabic, Devanagari)
//  KAZAKH (Latin, Cyrillic, Arabic)
//  KURDISH (Latin*, Arabic)
//  KYRGYZ (Cyrillic, Arabic)
//  LIMBU (Devanagari, Limbu)
//  MONGOLIAN (Cyrillic, Mongolian)
//  SANSKRIT (Latin, Devanagari)
//  SINDHI (Arabic, Devanagari)
//  TAGALOG (Latin, Tagalog)
//  TAJIK (Cyrillic, Arabic*)
//  TATAR (Latin, Cyrillic, Arabic)
//  TURKMEN (Latin, Cyrillic, Arabic)
//  UIGHUR (Latin, Cyrillic, Arabic)
//  UZBEK (Latin, Cyrillic, Arabic)
//
// * Due to a shortage of training text, AZERBAIJANI is not currently detected
//   in Arabic or Cyrillic scripts, nor KURDISH in Latin script, nor TAJIK in
//   Arabic script.
//

#ifndef ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_H_
#define ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_H_

#include "languages/public/languages.h"
#include "encodings/compact_lang_det/win/cld_utf8statetable.h"

namespace cld {
  struct CLDTableSummary;
}  // namespace cld

namespace CompactLangDet {
  // Scan interchange-valid UTF-8 bytes and detect most likely language,
  // or set of languages.
  //
  // Design goals:
  //   Skip over big stretches of HTML tags
  //   Able to return ranges of different languages
  //   Relatively small tables and relatively fast processing
  //   Thread safe
  //
  // For HTML documents, tags are skipped, along with <script> ... </script>
  // and <style> ... </style> sequences, and entities are expanded.
  //
  // We distinguish between bytes of the raw input buffer and bytes of non-tag
  // text letters. Since tags can be over 50% of the bytes of an HTML Page,
  // and are nearly all seven-bit ASCII English, we prefer to distinguish
  // language mixture fractions based on just the non-tag text.
  //
  // Inputs: text and text_length
  //  Code skips HTML tags and expands HTML entities, unless
  //  is_plain_text is true
  // Outputs:
  //  language3 is an array of the top 3 languages or UNKNOWN_LANGUAGE
  //  percent3 is an array of the text percentages 0..100 of the top 3 languages
  //  text_bytes is the amount of non-tag/letters-only text found
  //  is_reliable set true if the returned Language is some amount more
  //  probable then the second-best Language. Calculation is a complex function
  //  of the length of the text and the different-script runs of text.
  // Return value: the most likely Language for the majority of the input text
  //  Length 0 input returns UNKNOWN_LANGUAGE.
  //
  // Subsetting: For fast detection over large documents, these routines will
  // scan non-tag text of the initial part of a document, then will
  // skip 4-16 bytes and subsample text in the rest of the document, up to a
  // fixed limit (currently 160KB of non-tag letters).
  //

  struct DetectionTables {
    const cld::CLDTableSummary* quadgram_obj;
    const UTF8PropObj* unigram_obj;
  };

  // Scan interchange-valid UTF-8 bytes and detect list of top 3 languages.
  //
  // Accepts hints to bias languagepriors.
  //
  // Extended languages are additional Google interface languages and Unicode
  // single-language scripts, from ext_lang_enc.h. They are experimental and
  // this call may be removed.
  //
  // Returns internal language scores as a ratio to
  // normal score for real text in that language. Scores close to 1.0 indicate
  // normal text, while scores far away from 1.0 indicate badly-skewed text or
  // gibberish
  //
  // If do_pick_summary_lang is true then CLD will sometimes
  // not pick the top-scoring language; see CalcSummaryLang
  // in compact_lang_det_impl.cc.  If it's false then the
  // top language is always returned.
  //
  // If do_remove_weak_matches is true then CLD will delete
  // poor scoring languages from the results, so that if a
  // language is returned there is some confidence it is
  // correct.
  //
  Language DetectLanguage(
                          const DetectionTables* tables,
                          const char* buffer,
                          int buffer_length,
                          bool is_plain_text,
                          bool do_allow_extended_languages,
                          bool do_pick_summary_language,
                          bool do_remove_weak_matches,
                          const char* tld_hint,       // "id" boosts Indonesian
                          int encoding_hint,          // SJS boosts Japanese
                          Language language_hint,     // ITALIAN boosts it
                          Language* language3,
                          int* percent3,
                          double* normalized_score3,
                          int* text_bytes,
                          bool* is_reliable);

  // Return version text string
  // String is "code_version - data_scrape_date"
  const char* DetectLanguageVersion();
};      // End namespace CompactLangDet

#endif  // ENCODINGS_COMPACT_LANG_DET_COMPACT_LANG_DET_H_