Changeset 6142

Timestamp:

04/13/15 12:37:17 (9 years ago)

Author:

teckart@informatik.uni-leipzig.de

Message:

Added support for mapping language name variants to "official" language names (#749)

Location:

vlo/trunk/vlo-importer/src

Files:

• main/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessor.java (modified) (3 diffs)
• test/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessorTest.java (modified) (1 diff)

vlo/trunk/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessor.java

@@ -2 +2 @@
 
 import eu.clarin.cmdi.vlo.LanguageCodeUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
 
 public class LanguageCodePostProcessor implements PostProcessor{
 
     private final static Logger LOG = LoggerFactory.getLogger(LanguageCodePostProcessor.class);
+    
+    private static Map<String, String> languageNameVariantsMap = null;
 
     protected static final String CODE_PREFIX = "code:";
@@ -42 +60 @@
     protected String extractLanguageCode(String value) {
         final LanguageCodeUtils languageCodeUtils = MetadataImporter.languageCodeUtils;
-        
         String result = value;
         
         result = result.replaceFirst(ISO639_2_PREFIX, "").replaceFirst(ISO639_3_PREFIX, "").replaceFirst(SIL_CODE_PREFIX, "").replaceFirst(SIL_CODE_PREFIX_alt, "");
         
+        // map known language name variants to their offical name
+        if(getLanguageNamesVariantsMap().containsKey(result))
+            result = getLanguageNamesVariantsMap().get(result);
+        
         // input is already ISO 639-3?
-        if(languageCodeUtils.getIso639ToLanguageNameMap().keySet().contains(result.toUpperCase()))
+        if(languageCodeUtils.getIso639ToLanguageNameMap().keySet().contains(result.toUpperCase())) {
             return CODE_PREFIX + result.toLowerCase();
-        
+        }        
         // input is 2-letter code -> map to ISO 639-3
         if(languageCodeUtils.getSilToIso639Map().containsKey(result.toLowerCase())) {
@@ -75 +96 @@
         return result;
     }
+    
+        private Map<String, String> getLanguageNamesVariantsMap() {
+        if (languageNameVariantsMap == null) {
+            try {
+                // load records from file, in the future this should be loaded from CLAVAS directly and the file only used as fallback
+                languageNameVariantsMap = createControlledVocabularyMap(MetadataImporter.config.getLanguageNameVariantsUrl());
+            } catch (Exception e) {
+                throw new RuntimeException("Cannot instantiate postProcessor:", e);
+            }
+        }
+        return languageNameVariantsMap;
+    }
+
+    private Map<String, String> createControlledVocabularyMap(String languageNamesUrl) throws ParserConfigurationException, MalformedURLException, SAXException, XPathExpressionException, IOException {
+        Map<String, String> result = new HashMap<String, String>();
+        DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+        domFactory.setNamespaceAware(true);
+
+        InputStream mappingFileAsStream;
+        mappingFileAsStream = NationalProjectPostProcessor.class.getResourceAsStream(languageNamesUrl);
+
+        DocumentBuilder builder = domFactory.newDocumentBuilder();
+        Document doc = builder.parse(mappingFileAsStream);
+        XPath xpath = XPathFactory.newInstance().newXPath();
+        NodeList nodeList = (NodeList) xpath.evaluate("//Language", doc, XPathConstants.NODESET);
+        for (int i = 0; i < nodeList.getLength(); i++) {
+            Node node = nodeList.item(i);
+            String languageName = node.getAttributes().getNamedItem("name").getTextContent();
+            NodeList childNodeList = node.getChildNodes();
+            for (int j = 0; j < childNodeList.getLength(); j++) {
+                String variation = childNodeList.item(j).getTextContent();
+                result.put(variation, languageName);
+            }
+        }
+        return result;
+    }
 }

vlo/trunk/vlo-importer/src/test/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessorTest.java

@@ -38 +38 @@
         assertEquals("code:nld", processor.process("nl-NL").get(0));
         assertEquals("code:eng", processor.process("ISO639-2:eng").get(0));
+        assertEquals("code:spa", processor.process("Spanish, Castilian").get(0));
     }
 }