Changeset 6653

Timestamp:

10/07/15 10:51:24 (9 years ago)

Author:

teckart@informatik.uni-leipzig.de

Message:

More consistent handling of xml:lang attributes for description values. Now the same schema as for facet languageCode is used to indicate the content's language (#780)

Location:

vlo/trunk/vlo-importer/src

Files:

• main/java/eu/clarin/cmdi/vlo/importer/CMDIParserVTDXML.java (modified) (4 diffs)
• test/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessorTest.java (modified) (4 diffs)

vlo/trunk/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/CMDIParserVTDXML.java

@@ -29 +29 @@
     private final static Logger LOG = LoggerFactory.getLogger(CMDIParserVTDXML.class);
 
-    private static final String DEFAULT_LANGUAGE = "und";
+    private static final String DEFAULT_LANGUAGE = "code:und";
 
     public CMDIParserVTDXML(Map<String, PostProcessor> postProcessors, Boolean useLocalXSDCache) {
@@ -285 +285 @@
 
             // ignore non-English language names for facet LANGUAGE_CODE
-            if (config.getName().equals(FacetConstants.FIELD_LANGUAGE_CODE) && !languageCode.equals("en") && !languageCode.equals("eng") && !languageCode.equals("und")) {
+            if (config.getName().equals(FacetConstants.FIELD_LANGUAGE_CODE) && !languageCode.equals("code:eng") && !languageCode.equals("code:und")) {
                 index = ap.evalXPath();
                 continue;
@@ -314 +314 @@
         // extract language code in xml:lang if available
         Integer langAttrIndex = nav.getAttrVal("xml:lang");
-        String languageCode = DEFAULT_LANGUAGE;
+        String languageCode;
         if (langAttrIndex != -1) {
             languageCode = nav.toString(langAttrIndex).trim();
-        }
-        // replace 2-letter with 3-letter codes
-        if (MetadataImporter.languageCodeUtils.getSilToIso639Map().containsKey(languageCode)) {
-            languageCode = MetadataImporter.languageCodeUtils.getSilToIso639Map().get(languageCode);
-        }
-        return languageCode;
+        } else {
+            return DEFAULT_LANGUAGE;
+        }
+
+        return postProcessors.get(FacetConstants.FIELD_LANGUAGE_CODE).process(languageCode).get(0);
     }
 
@@ -332 +331 @@
             String fieldValue = valueList.get(i).trim();
             if (name.equals(FacetConstants.FIELD_DESCRIPTION)) {
-                fieldValue = "{lang='" + languageCode + "'}" + fieldValue;
+                fieldValue = "{" + languageCode + "}" + fieldValue;
             }
             cmdiData.addDocField(name, fieldValue, caseInsensitive);

vlo/trunk/vlo-importer/src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessorTest.java

@@ -402 +402 @@
         assertEquals("demo", doc.getFieldValue("genre"));
         assertEquals(
-                "{lang='eng'}This  recording was made to generate a freely available test resource including speech and gestures. The annotations were created by Peter and Kita who is gesture researcher at the MPI for Psycholinguistics.",
+                "{code:eng}This  recording was made to generate a freely available test resource including speech and gestures. The annotations were created by Peter and Kita who is gesture researcher at the MPI for Psycholinguistics.",
                 doc.getFieldValue("description"));
         assertEquals("2002-10-30", doc.getFieldValue("temporalCoverage"));
@@ -519 +519 @@
         assertEquals("Netherlands", doc.getFieldValue("country"));
         assertEquals("demo", doc.getFieldValue("genre"));
-        assertEquals("{lang='und'}Test.", doc.getFieldValue("description"));
+        assertEquals("{code:und}Test.", doc.getFieldValue("description"));
         assertEquals("Should be null not empty string", null, doc.getFieldValue("organisation"));
         assertEquals(null, doc.getFieldValue("language"));
@@ -597 +597 @@
         List<String> descriptions = new ArrayList(fieldValues);
         Collections.sort(descriptions);
-        assertEquals("{lang='und'}Channel: Talking;\n    Genre: Traditional Narrative / Story;\n    Country: Panama;\n"
+        assertEquals("{code:und}Channel: Talking;\n    Genre: Traditional Narrative / Story;\n    Country: Panama;\n"
                 + "    Place of Recording: Mulatuppu;\n    Event: Community Gathering;\n"
                 + "    Institutional Affiliation: University of Texas at Austin;\n    Participant Information: Political Leader;", descriptions.get(0).toString());
-        assertEquals("{lang='und'}Test", descriptions.get(1).toString());
-        assertEquals("{lang='und'}The one-eyed grandmother is one of many traditional Kuna stories performed "
+        assertEquals("{code:und}Test", descriptions.get(1).toString());
+        assertEquals("{code:und}The one-eyed grandmother is one of many traditional Kuna stories performed "
                 + "in the Kuna gathering house. This story, performed here by Pedro Arias, combines "
                 + "European derived motifs (Tom Thumb and Hansel and Gretel) with themes that seem more "
@@ -878 +878 @@
         assertEquals(null, doc.getFieldValue("year"));
         assertEquals(null, doc.getFieldValue("genre"));
-        assertEquals("{lang='eng'}written general; 95 mio words; TEI/SGML", doc.getFieldValue("description"));
+        assertEquals("{code:eng}written general; 95 mio words; TEI/SGML", doc.getFieldValue("description"));
         assertEquals("Written Corpus", doc.getFieldValue(FacetConstants.FIELD_RESOURCE_CLASS));
     }