Changeset 4575

Timestamp:

02/25/14 15:05:49 (10 years ago)

Author:

teckart

Message:

Attempt to fix ticket #412: "Allow for plain <language> tagging" by adding language DCMI to facet concepts and supporting ISO639-2/B language codes

Location:

vlo/branches/vlo-3.0

Files:

• vlo-commons/src/main/resources/facetConcepts.xml (modified) (2 diffs)
• vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessor.java (modified) (3 diffs)
• vlo-importer/src/test/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessorTest.java (modified) (1 diff)

vlo/branches/vlo-3.0/vlo-commons/src/main/resources/facetConcepts.xml

@@ -94 +94 @@
     <concept>http://www.isocat.org/datcat/DC-5361</concept> <!-- language usage (TEI) -->
     <concept>http://www.isocat.org/datcat/DC-5358</concept> <!--language (TEI) - MENZO: added for CLARIN-DK -->
+    
+    <concept>http://purl.org/dc/terms/language</concept>
    <!-- Had to be removed due to IDSAGD_Speaker  <acceptableContext includeAny="false" includeEmpty="true"/>
     <rejectableContext includeAny="true"  includeEmpty="false">
@@ -126 +128 @@
     <concept>http://www.isocat.org/datcat/DC-5361</concept> <!-- language usage (TEI) -->
     <concept>http://www.isocat.org/datcat/DC-5358</concept> <!--language (TEI) - MENZO: added for CLARIN-DK -->
+    
+    <concept>http://purl.org/dc/terms/language</concept>
    <!-- Had to be removed due to IDSAGD_Speaker <acceptableContext includeAny="false" includeEmpty="true"/>
     <rejectableContext includeAny="true"  includeEmpty="false">

vlo/branches/vlo-3.0/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessor.java

@@ -32 +32 @@
     private Map<String, String> languageNameToIso639Map;
     private Map<String, String> iso639ToLanguageNameMap;
+    private Map<String, String> iso639_2TToISO639_3Map;
 
     /**
@@ -73 +74 @@
             }
         }
+        
+        // convert ISO 639-2/T codes to ISO 639-3
+        if (getIso6392TToISO6393Map().containsKey(value.toUpperCase())) {
+            result = getIso6392TToISO6393Map().get(value.toUpperCase());
+        }
+        
         // Convert to lowercase to capture erroneously capitalized language codes in the CMDI files.
         // NOTE: In the mappings themselves we do not capitalize.
@@ -122 +129 @@
 
         return iso639ToLanguageNameMap;
+    }
+    
+    /**
+     * Returns map of ISO 639-2/B codes to ISO 639-3
+     * 
+     *  It is strongly advised to use ISO 639-3 codes, the support for ISO 639-2 may be discontinued in the future
+     * 
+     * @return map of ISO 639-2/B codes to ISO 639-3
+     */
+    private Map<String, String> getIso6392TToISO6393Map() {
+        if (iso639_2TToISO639_3Map == null) {
+            iso639_2TToISO639_3Map = new HashMap<String, String>();
+            iso639_2TToISO639_3Map.put("ALB", "SQI");
+            iso639_2TToISO639_3Map.put("ARM", "HYE");
+            iso639_2TToISO639_3Map.put("BAQ", "EUS");
+            iso639_2TToISO639_3Map.put("BUR", "MYA");
+            iso639_2TToISO639_3Map.put("CZE", "CES");
+            iso639_2TToISO639_3Map.put("CHI", "ZHO");
+            iso639_2TToISO639_3Map.put("DUT", "NLD");
+            iso639_2TToISO639_3Map.put("FRE", "FRA");
+            iso639_2TToISO639_3Map.put("GEO", "KAT");
+            iso639_2TToISO639_3Map.put("GER", "DEU");
+            iso639_2TToISO639_3Map.put("GRE", "ELL");
+            iso639_2TToISO639_3Map.put("ICE", "ISL");
+            iso639_2TToISO639_3Map.put("MAC", "MKD");
+            iso639_2TToISO639_3Map.put("MAO", "MRI");
+            iso639_2TToISO639_3Map.put("MAY", "MSA");
+            iso639_2TToISO639_3Map.put("PER", "FAS");
+            iso639_2TToISO639_3Map.put("RUM", "RON");
+            iso639_2TToISO639_3Map.put("SLO", "SLK");
+            iso639_2TToISO639_3Map.put("TIB", "BOD");
+            iso639_2TToISO639_3Map.put("WEL", "CYM");
+        }
+        
+        return iso639_2TToISO639_3Map;
     }
 

vlo/branches/vlo-3.0/vlo-importer/src/test/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessorTest.java

@@ -30 +30 @@
         assertEquals("German", processor.process("RFC1766:x-sil-GER"));
         assertEquals("RFC1766:sgn-NL", processor.process("RFC1766:sgn-NL"));
+        assertEquals("Basque", processor.process("baq"));
     }