adds language extractor feature

Author: getorca

.gitignore

@@ -29,6 +29,7 @@ docs/_template/
 
 # pipenv
 Pipfile*
+venv/
 
 # older stuff
 old/

trafilatura/core.py

@@ -27,7 +27,7 @@
 # own
 from .external import justext_rescue, sanitize_tree, SANITIZED_XPATH, try_readability
 from .filters import (LANGID_FLAG, check_html_lang, content_fingerprint, duplicate_test,
-                      language_filter, text_chars_test)
+                      language_filter, text_chars_test, extract_lang)
 from .htmlprocessing import (convert_tags, handle_textnode, process_node,
                              delete_by_link_density, link_density_test_tables,
                              prune_unwanted_nodes, tree_cleaning)
@@ -950,6 +950,10 @@ def bare_extraction(filecontent, url=None, no_fallback=False,  # fast=False,
             LOGGER.error('duplicate document for URL %s', url)
             raise ValueError
 
+
+        # extract lang
+        document.language = extract_lang(tree_backup_2, temp_text, temp_comments)
+
         # sanity check on language
         if target_language is not None:
             is_not_target_lang, document = language_filter(temp_text, temp_comments, target_language, document)

trafilatura/filters.py

@@ -60,6 +60,31 @@ def duplicate_test(element, config):
     put_in_cache(teststring)
     return False
 
+def extract_lang(tree, temp_text, temp_comments, strict=False):
+    """
+    extracts the language from HTML meta-elements and the language_classifier if library installed
+    returns the first language extracted
+    """
+    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Language
+    elems = set()
+    target_elements = tree.findall('.//meta[@http-equiv="content-language"][@content]')
+    if target_elements:
+        elems.update([x.get('content').lower()[:2] for x in target_elements])
+    # locale
+    target_elements = tree.findall('.//meta[@property="og:locale"][@content]')
+    if target_elements:
+        elems.update([x.get('content').lower()[:2] for x in target_elements])
+    # HTML lang attribute: sometimes a wrong indication
+    if strict is True:
+        target_elements = tree.xpath('//html[@lang]')
+        if target_elements:
+            elems.update([x.get('content').lower()[:2] for x in target_elements])
+    # external tool
+    classified_lang = language_classifier(temp_text, temp_comments)
+    if classified_lang:
+        elems.add(classified_lang)
+    return [*elems, ][0]
+
 
 def check_html_lang(tree, target_language, strict=False):
     '''Check HTML meta-elements for language information and split