Changeset 6760

Timestamp:

11/09/15 13:49:39 (9 years ago)

Author:

davor.ostojic@oeaw.ac.at

Message:

merged with trunk 3.4

Location:

vlo/branches/vlo-3.3-oeaw/vlo-importer

Files:

• . (modified) (1 prop)
• pom.xml (modified) (2 diffs)
• src/main/java/eu/clarin/cmdi/vlo/importer/CMDIParserVTDXML.java (modified) (4 diffs)
• src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java (modified) (6 diffs)
• src/main/java/eu/clarin/cmdi/vlo/importer/MetadataMapper.java (copied) (copied from vlo/trunk/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/MetadataMapper.java)
• src/main/java/eu/clarin/cmdi/vlo/importer/ResourceStructureGraph.java (modified) (7 diffs)
• src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessorTest.java (modified) (4 diffs)
• src/test/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessorTest.java (modified) (4 diffs)

vlo/branches/vlo-3.3-oeaw/vlo-importer/pom.xml

@@ -7 +7 @@
         <groupId>eu.clarin.cmdi</groupId>
         <artifactId>vlo</artifactId>
-        <version>3.3-SNAPSHOT</version>
+        <version>3.4-SNAPSHOT</version>
     </parent>
     
@@ -13 +13 @@
     <groupId>eu.clarin.cmdi</groupId>
     <artifactId>vlo-importer</artifactId> 
-    <version>3.3-SNAPSHOT</version>
+    <version>3.4-SNAPSHOT</version>
     <packaging>jar</packaging>
 

vlo/branches/vlo-3.3-oeaw/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/CMDIParserVTDXML.java

@@ -36 +36 @@
     private final static Logger LOG = LoggerFactory.getLogger(CMDIParserVTDXML.class);
 
-    private static final String DEFAULT_LANGUAGE = "und";
+    private static final String DEFAULT_LANGUAGE = "code:und";
 
     public CMDIParserVTDXML(Map<String, PostProcessor> postProcessors, Boolean useLocalXSDCache) {
@@ -291 +291 @@
 
             // ignore non-English language names for facet LANGUAGE_CODE
-            if (config.getName().equals(FacetConstants.FIELD_LANGUAGE_CODE) && !languageCode.equals("en") && !languageCode.equals("eng") && !languageCode.equals("und")) {
+            if (config.getName().equals(FacetConstants.FIELD_LANGUAGE_CODE) && !languageCode.equals("code:eng") && !languageCode.equals("code:und")) {
                 index = ap.evalXPath();
                 continue;
@@ -329 +329 @@
         // extract language code in xml:lang if available
         Integer langAttrIndex = nav.getAttrVal("xml:lang");
-        String languageCode = DEFAULT_LANGUAGE;
+        String languageCode;
         if (langAttrIndex != -1) {
             languageCode = nav.toString(langAttrIndex).trim();
-        }
-        // replace 2-letter with 3-letter codes
-        if (MetadataImporter.languageCodeUtils.getSilToIso639Map().containsKey(languageCode)) {
-            languageCode = MetadataImporter.languageCodeUtils.getSilToIso639Map().get(languageCode);
-        }
-        return languageCode;
+        } else {
+            return DEFAULT_LANGUAGE;
+        }
+
+        return postProcessors.get(FacetConstants.FIELD_LANGUAGE_CODE).process(languageCode).get(0);
     }
 
@@ -363 +362 @@
             String fieldValue = valueList.get(i).trim();
             if (name.equals(FacetConstants.FIELD_DESCRIPTION)) {
-                fieldValue = "{lang='" + languageCode + "'}" + fieldValue;
+                fieldValue = "{" + languageCode + "}" + fieldValue;
             }
             cmdiData.addDocField(name, fieldValue, caseInsensitive);

vlo/branches/vlo-3.3-oeaw/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java

@@ -48 +48 @@
  * and so on.
  */
-@SuppressWarnings({"serial"})
 public class MetadataImporter {
 
@@ -166 +165 @@
                     }
                     solrServer.commit();
-                    updateDocumentHierarchy();
+                    if(config.isProcessHierarchies()){
+                        updateDocumentHierarchy();
+                    }
                 }
                 LOG.info("End of processing: " + dataRoot.getOriginName());
@@ -227 +228 @@
      * list of existing data roots will be filtered with the list from user
      * 
-     * @return
+     * @param dataRoots complete list of DataRoots
+     * @return list of DataRoots without DataRoots excluded by the user
      */
     protected List<DataRoot> filterDataRootsWithCLArgs(List<DataRoot> dataRoots){
@@ -364 +366 @@
                         // are usually not very interesting (think imdi corpus files) and will not be included.
                         updateDocument(solrDocument, cmdiData, file, dataOrigin);
+                        if(ResourceStructureGraph.getVertex(cmdiData.getId()) != null)
+                            ResourceStructureGraph.getVertex(cmdiData.getId()).setWasImported(true);
                     } else {
                         nrOfIgnoredFiles++;
@@ -538 +542 @@
                 // get document
                 query = new SolrQuery();
+                query.setRequestHandler(FacetConstants.SOLR_REQUEST_HANDLER_FAST);
                 query.set("q", FacetConstants.FIELD_ID+":"+vertex.getId());
                 SolrDocumentList response = solrServer.query(query).getResults();
@@ -550 +555 @@
                 if(vertex.getHierarchyWeight() != 0) {
                     doc.setField(FacetConstants.FIELD_HIERARCHY_WEIGHT, Math.abs(vertex.getHierarchyWeight()));
+                }
+                
+                // remove vertices that were not imported
+                Iterator<String> incomingVertexIter = incomingVertexNames.iterator();
+                while(incomingVertexIter.hasNext()) {
+                    String vertexId = incomingVertexIter.next();
+                    if(ResourceStructureGraph.getVertex(vertexId) == null || !ResourceStructureGraph.getVertex(vertexId).getWasImported())
+                        incomingVertexIter.remove();
+                }
+                Iterator<String> outgoingVertexIter = outgoingVertexNames.iterator();
+                while(outgoingVertexIter.hasNext()) {
+                    String vertexId = outgoingVertexIter.next();
+                    if(ResourceStructureGraph.getVertex(vertexId) == null || !ResourceStructureGraph.getVertex(vertexId).getWasImported())
+                        outgoingVertexIter.remove();
                 }
                 

vlo/branches/vlo-3.3-oeaw/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/ResourceStructureGraph.java

@@ -53 +53 @@
      */
     public static void addResource(String mdSelfLink) {
-        if (!vertexIdMap.containsKey(mdSelfLink)) {
-            CmdiVertex newVertex = new CmdiVertex(StringUtils.normalizeIdString(mdSelfLink));
-            vertexIdMap.put(mdSelfLink, newVertex);
-        }
-
-        if (!foundVerticesSet.contains(vertexIdMap.get(mdSelfLink))) {
-            graph.addVertex(vertexIdMap.get(mdSelfLink));
-            foundVerticesSet.add(vertexIdMap.get(mdSelfLink));
+        String normalizedMdSelfLink = StringUtils.normalizeIdString(mdSelfLink);
+        if (!vertexIdMap.containsKey(normalizedMdSelfLink)) {
+            CmdiVertex newVertex = new CmdiVertex(normalizedMdSelfLink);
+            vertexIdMap.put(normalizedMdSelfLink, newVertex);
+        }
+
+        if (!foundVerticesSet.contains(vertexIdMap.get(normalizedMdSelfLink))) {
+            graph.addVertex(vertexIdMap.get(normalizedMdSelfLink));
+            foundVerticesSet.add(vertexIdMap.get(normalizedMdSelfLink));
         } else {
             LOG.debug("Duplicate resource vertex mdSelfLink: " + mdSelfLink);
@@ -73 +74 @@
      */
     public static void addEdge(String sourceVertexId, String targetVertexId) {
+        String normalizedSourceVertexId = StringUtils.normalizeIdString(sourceVertexId);
+        String normalizedTargetVertexId = StringUtils.normalizeIdString(targetVertexId);
+        
         // add vertices
-        if (!vertexIdMap.containsKey(sourceVertexId)) {
-            CmdiVertex sourceVertex = new CmdiVertex(StringUtils.normalizeIdString(sourceVertexId));
-            vertexIdMap.put(sourceVertexId, sourceVertex);
+        if (!vertexIdMap.containsKey(normalizedSourceVertexId)) {
+            CmdiVertex sourceVertex = new CmdiVertex(normalizedSourceVertexId);
+            vertexIdMap.put(normalizedSourceVertexId, sourceVertex);
             graph.addVertex(sourceVertex);
         }
 
-        if (!vertexIdMap.containsKey(targetVertexId)) {
-            CmdiVertex targetVertex = new CmdiVertex(StringUtils.normalizeIdString(targetVertexId));
-            vertexIdMap.put(targetVertexId, targetVertex);
+        if (!vertexIdMap.containsKey(normalizedTargetVertexId)) {
+            CmdiVertex targetVertex = new CmdiVertex(normalizedTargetVertexId);
+            vertexIdMap.put(normalizedTargetVertexId, targetVertex);
             graph.addVertex(targetVertex);
         }
@@ -88 +92 @@
         // add edge
         try {
-            graph.addEdge(vertexIdMap.get(sourceVertexId), vertexIdMap.get(targetVertexId));
-            updateDepthValues(vertexIdMap.get(sourceVertexId), new HashSet<CmdiVertex>());
+            graph.addEdge(vertexIdMap.get(normalizedSourceVertexId), vertexIdMap.get(normalizedTargetVertexId));
+            updateDepthValues(vertexIdMap.get(normalizedSourceVertexId), new HashSet<CmdiVertex>());
         } catch (IllegalArgumentException cfe) {
             // was a cycle -> ignore
@@ -162 +166 @@
         return foundVerticesSet;
     }
+    
+    public static CmdiVertex getVertex(String vertexId) {
+        return vertexIdMap.get(vertexId);        
+    }
+    
+    public static Map<String, CmdiVertex> getVertexIdMap() {
+        return vertexIdMap;
+    }
 
     /**
      * Get all vertices that are source of an edge where targetVertex is target.
-     * In other words get all vertices that are part of targetVertex.
+     * In other words get all resource vertices that are part of resource targetVertex.
      *
      * @param targetVertex
@@ -187 +199 @@
     /**
      * Get all vertices that are target of an edge where sourceVertex is source.
-     * In other words get all vertices of which sourceVertex is part of.
+     * In other words get all resource vertices of which resource sourceVertex is part of.
      *
      * @param sourceVertex
@@ -273 +285 @@
     private final String id;
     private int hierarchyWeight = 0;
+    private boolean wasImported = false;
 
     public CmdiVertex(String id) {
@@ -288 +301 @@
     public int getHierarchyWeight() {
         return hierarchyWeight;
+    }
+    
+    public void setWasImported(boolean wasImported) {
+        this.wasImported = wasImported;
+    }
+    
+    public boolean getWasImported() {
+        return wasImported;
     }
 

vlo/branches/vlo-3.3-oeaw/vlo-importer/src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessorTest.java

@@ -402 +402 @@
         assertEquals("demo", doc.getFieldValue("genre"));
         assertEquals(
-                "{lang='eng'}This  recording was made to generate a freely available test resource including speech and gestures. The annotations were created by Peter and Kita who is gesture researcher at the MPI for Psycholinguistics.",
+                "{code:eng}This  recording was made to generate a freely available test resource including speech and gestures. The annotations were created by Peter and Kita who is gesture researcher at the MPI for Psycholinguistics.",
                 doc.getFieldValue("description"));
         assertEquals("2002-10-30", doc.getFieldValue("temporalCoverage"));
@@ -519 +519 @@
         assertEquals("Netherlands", doc.getFieldValue("country"));
         assertEquals("demo", doc.getFieldValue("genre"));
-        assertEquals("{lang='und'}Test.", doc.getFieldValue("description"));
+        assertEquals("{code:und}Test.", doc.getFieldValue("description"));
         assertEquals("Should be null not empty string", null, doc.getFieldValue("organisation"));
         assertEquals(null, doc.getFieldValue("language"));
@@ -597 +597 @@
         List<String> descriptions = new ArrayList(fieldValues);
         Collections.sort(descriptions);
-        assertEquals("{lang='und'}Channel: Talking;\n    Genre: Traditional Narrative / Story;\n    Country: Panama;\n"
+        assertEquals("{code:und}Channel: Talking;\n    Genre: Traditional Narrative / Story;\n    Country: Panama;\n"
                 + "    Place of Recording: Mulatuppu;\n    Event: Community Gathering;\n"
                 + "    Institutional Affiliation: University of Texas at Austin;\n    Participant Information: Political Leader;", descriptions.get(0).toString());
-        assertEquals("{lang='und'}Test", descriptions.get(1).toString());
-        assertEquals("{lang='und'}The one-eyed grandmother is one of many traditional Kuna stories performed "
+        assertEquals("{code:und}Test", descriptions.get(1).toString());
+        assertEquals("{code:und}The one-eyed grandmother is one of many traditional Kuna stories performed "
                 + "in the Kuna gathering house. This story, performed here by Pedro Arias, combines "
                 + "European derived motifs (Tom Thumb and Hansel and Gretel) with themes that seem more "
@@ -879 +879 @@
         assertEquals(null, doc.getFieldValue("year"));
         assertEquals(null, doc.getFieldValue("genre"));
-        assertEquals("{lang='eng'}written general; 95 mio words; TEI/SGML", doc.getFieldValue("description"));
+        assertEquals("{code:eng}written general; 95 mio words; TEI/SGML", doc.getFieldValue("description"));
         assertEquals("corpus", doc.getFieldValue(FacetConstants.FIELD_RESOURCE_CLASS));
         //assertEquals("Written Corpus", doc.getFieldValue(FacetConstants.FIELD_RESOURCE_CLASS));

vlo/branches/vlo-3.3-oeaw/vlo-importer/src/test/java/eu/clarin/cmdi/vlo/importer/LanguageCodePostProcessorTest.java

@@ -1 +1 @@
 package eu.clarin.cmdi.vlo.importer;
 
+import eu.clarin.cmdi.vlo.config.DefaultVloConfigFactory;
 import static org.junit.Assert.assertEquals;
-
+import org.junit.Before;
 import org.junit.Test;
 
 public class LanguageCodePostProcessorTest extends ImporterTestcase {
+
+    @Before
+    public void setUp() throws Exception {
+
+        // read the configuration from the packaged configuration file
+        new DefaultVloConfigFactory().newConfig();
+
+        // optionally, modify the configuration here
+    }
 
     @Test
@@ -12 +22 @@
         assertEquals("code:nld", processor.process("NL").get(0));
         assertEquals("code:eng", processor.process("en").get(0));
-        assertEquals("code:nld", processor.process("nl").get(0));
         assertEquals("code:fry", processor.process("fry").get(0));
         assertEquals("name:test", processor.process("test").get(0));
@@ -23 +32 @@
         assertEquals("code:eng", processor.process("eng").get(0));
         assertEquals("code:eng", processor.process("English").get(0));
-        assertEquals("code:deu", processor.process("German").get(0));
         assertEquals("code:esn", processor.process("Salvadoran Sign Language").get(0));
         assertEquals("code:eng", processor.process("en_US").get(0));
@@ -29 +37 @@
         assertEquals("code:eng", processor.process("ISO639-2:eng").get(0));
         assertEquals("code:spa", processor.process("Spanish, Castilian").get(0));
-        
-        
+        assertEquals("code:ron", processor.process("Romanian").get(0));
     }
 }