Changeset 996

Timestamp:

12/15/10 12:48:10 (13 years ago)

Author:

patdui

Message:

• Added vtd-xml parser which is 10x faster (5mins vs. 1hour for 200.000 cmdi files) then DOM parser. Still supports xpath to query the data out.
• Made it possible to delete only part of the index, you can delete a whole origin.

Location:

vlo/trunk/vlo_webapp

Files:

• pom.xml (modified) (1 diff)
• src/main/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessor.java (added)
• src/main/java/eu/clarin/cmdi/vlo/importer/CMDIDigester.java (modified) (2 diffs)
• src/main/java/eu/clarin/cmdi/vlo/importer/CMDIParserVTDXML.java (added)
• src/main/java/eu/clarin/cmdi/vlo/importer/DataRoot.java (modified) (2 diffs)
• src/main/java/eu/clarin/cmdi/vlo/importer/ImporterConfig.java (modified) (2 diffs)
• src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java (modified) (8 diffs)
• src/main/resources/importerConfig.xml (modified) (2 diffs)
• src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessorTest.java (moved) (moved from vlo/trunk/vlo_webapp/src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDigesterTest.java) (10 diffs)

vlo/trunk/vlo_webapp/pom.xml

@@ -76 +76 @@
 
     <dependency>
-      <groupId>commons-digester</groupId>
-      <artifactId>commons-digester</artifactId>
-      <version>2.1</version>
+         <groupId>com.ximpleware</groupId>
+         <artifactId>vtd-xml</artifactId>
+         <version>2.9</version>
     </dependency>
   </dependencies>

vlo/trunk/vlo_webapp/src/main/java/eu/clarin/cmdi/vlo/importer/CMDIDigester.java

@@ -14 +14 @@
 import javax.xml.xpath.XPathFactory;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.w3c.dom.Document;
 import org.w3c.dom.Node;
@@ -22 +20 @@
 import org.xml.sax.SAXException;
 
-public class CMDIDigester {
-    private final static Logger LOG = LoggerFactory.getLogger(CMDIDigester.class);
+
+/**
+ * @deprecated 
+ * Dom parsing implementation, use the @see CMDIParserVTDXML it is much faster. 
+ * Keeping this for now just in case we run into issues with the vlt parsing.
+ * patdui 15 December 2010
+ */
+public class CMDIDigester implements CMDIDataProcessor {
     private final FacetMapping facetMapping;
     private DocumentBuilder builder;

vlo/trunk/vlo_webapp/src/main/java/eu/clarin/cmdi/vlo/importer/DataRoot.java

@@ -8 +8 @@
     private String originName;
     private File rootFile;
+    private boolean deleteFirst = false;
 
     public void setFacetMapping(FacetMapping facetMapping) {
@@ -33 +34 @@
     }
 
+    public void setDeleteFirst(boolean deleteFirst) {
+        this.deleteFirst = deleteFirst;
+    }
+
+    public boolean isDeleteFirst() {
+        return deleteFirst;
+    }
+
 }

vlo/trunk/vlo_webapp/src/main/java/eu/clarin/cmdi/vlo/importer/ImporterConfig.java

@@ -5 +5 @@
 public class ImporterConfig {
 
-    private boolean deleteFirst = false;
+    private boolean deleteAllFirst = false;
     
     private List<DataRoot> dataRoots;
@@ -17 +17 @@
     }
 
-    public void setDeleteFirst(boolean deleteFirst) {
-        this.deleteFirst = deleteFirst;
+    public void setDeleteAllFirst(boolean deleteAllFirst) {
+        this.deleteAllFirst = deleteAllFirst;
     }
 
-    public boolean isDeleteFirst() {
-        return deleteFirst;
+    public boolean isDeleteAllFirst() {
+        return deleteAllFirst;
     }
 

vlo/trunk/vlo_webapp/src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java

@@ -9 +9 @@
 import java.util.Set;
 
-import javax.xml.xpath.XPathExpressionException;
-
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.StreamingUpdateSolrServer;
@@ -18 +16 @@
 import org.springframework.beans.factory.BeanFactory;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.xml.sax.SAXException;
 
 import eu.clarin.cmdi.vlo.Configuration;
@@ -52 +49 @@
     }
 
-    //TODO PD can have multiple origins
     private void startImport() {
         List<DataRoot> dataRoots = config.getDataRoots();
@@ -64 +60 @@
         long start = System.currentTimeMillis();
         try {
-            if (config.isDeleteFirst()) {
+            if (config.isDeleteAllFirst()) {
                 LOG.info("Deleting original data...");
                 solrServer.deleteByQuery("*:*");//Delete the whole solr db.
@@ -70 +66 @@
             }
             for (DataRoot dataRoot : dataRoots) {
-                LOG.info("Start of processing: "+dataRoot.getOriginName());
-                CMDIDigester digester = new CMDIDigester(dataRoot.getFacetMapping());
-                processCmdi(dataRoot.getRootFile(), dataRoot.getOriginName(), digester);
+                LOG.info("Start of processing: " + dataRoot.getOriginName());
+                if (dataRoot.isDeleteFirst()) {
+                    LOG.info("Deleting data for origin: " + dataRoot.getOriginName());
+                    solrServer.deleteByQuery(FacetConstants.FIELD_ORIGIN + ":" + dataRoot.getOriginName());
+                    LOG.info("Deleting data for origin done.");
+                }
+                CMDIDataProcessor processor = new CMDIParserVTDXML(dataRoot.getFacetMapping());
+                processCmdi(dataRoot.getRootFile(), dataRoot.getOriginName(), processor);
                 if (!docs.isEmpty()) {
                     sendDocs();
                 }
-                LOG.info("End of processing: "+dataRoot.getOriginName());
+                LOG.info("End of processing: " + dataRoot.getOriginName());
             }
         } catch (SolrServerException e) {
@@ -98 +99 @@
     }
 
-    private void processCmdi(File file, String origin, CMDIDigester digester) throws SolrServerException, IOException {
+    private void processCmdi(File file, String origin, CMDIDataProcessor processor) throws SolrServerException, IOException {
         nrOfFilesAnalyzed++;
         CMDIData cmdiData = null;
         try {
-            cmdiData = digester.process(file);
-        } catch (IOException e) {
-            LOG.error("error in file: " + file + " Exception", e);
-        } catch (SAXException e) {
-            LOG.error("error in file: " + file + " Exception", e);
-        } catch (XPathExpressionException e) {
+            cmdiData = processor.process(file);
+        } catch (Exception e) {
             LOG.error("error in file: " + file + " Exception", e);
         }
@@ -119 +116 @@
                 File resourceFile = new File(file.getParentFile(), cmdiResource);
                 if (resourceFile.exists()) {
-                    processCmdi(resourceFile, origin, digester);
+                    processCmdi(resourceFile, origin, processor);
                 } else {
                     nrOfNonExistendResourceFiles++;
@@ -146 +143 @@
 
     private void sendDocs() throws SolrServerException, IOException {
-        LOG.info("Sending "+docs.size()+" docs to solr server. Total number of docs updated till now: "+nrOFDocumentsUpdated);
+        LOG.info("Sending " + docs.size() + " docs to solr server. Total number of docs updated till now: " + nrOFDocumentsUpdated);
         nrOFDocumentsUpdated += docs.size();
         solrServer.add(docs);

vlo/trunk/vlo_webapp/src/main/resources/importerConfig.xml

@@ -4 +4 @@
 
   <bean id="importerConfig" class="eu.clarin.cmdi.vlo.importer.ImporterConfig">
-    <!--    <property name="deleteFirst" value="true"/>-->
+    <property name="deleteAllFirst" value="true"/>
     <property name="dataRoots">
       <list>
         <bean class="eu.clarin.cmdi.vlo.importer.DataRoot">
+<!--          <property name="deleteFirst"  value="true"/>-->
           <property name="originName" value="OLAC Metadata Providers" />
           <property name="rootFile"
@@ -16 +17 @@
           <property name="originName" value="Nijmegen corpora of casual speech" />
           <property name="rootFile"
-
             value="/Users/patdui/data/snapshots2/data/corpora/qfs1/media-archive/casual_speech/Corpusstructure/casual_speech.imdi.cmdi" />
           <property name="facetMapping" ref="imdiMapping"></property>

vlo/trunk/vlo_webapp/src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessorTest.java

@@ -7 +7 @@
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
@@ -17 +20 @@
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
-public class CMDIDigesterTest {
+public class CMDIDataProcessorTest {
 
     private static File testDir;
+
+    private CMDIDataProcessor getDataParser(FacetMapping map) {
+        return new CMDIParserVTDXML(map);
+        //        return new CMDIDigester(map);
+    }
 
     @Test
@@ -66 +74 @@
         content += "</CMD>\n";
         File cmdiFile = createCmdiFile("testCorpus", content);
-        CMDIDigester digester = new CMDIDigester(getIMDIFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getIMDIFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("test-hdl:1839/00-0000-0000-0000-0001-D", data.getId());
         List<String> resources = data.getResources();
@@ -362 +370 @@
         content += "</CMD>\n";
         File cmdiFile = createCmdiFile("testSession", content);
-        CMDIDigester digester = new CMDIDigester(getIMDIFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getIMDIFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("test-hdl:1839/00-0000-0000-0009-294C-9", data.getId());
         List<String> resources = data.getResources();
@@ -452 +460 @@
         content += "</CMD>\n";
         File cmdiFile = createCmdiFile("testSession", content);
-        CMDIDigester digester = new CMDIDigester(getIMDIFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getIMDIFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("test-hdl:1839/00-0000-0000-0009-294C-9", data.getId());
         List<String> resources = data.getResources();
@@ -505 +513 @@
         content += "         <description>The one-eyed grandmother is one of many traditional Kuna stories performed in the Kuna gathering house. This story, performed here by Pedro Arias, combines European derived motifs (Tom Thumb and Hansel and Gretel) with themes that seem more Kuna in origin. All are woven together and a moral is provided. Pedro Arias performed this story before a gathered audience in the morning..\n";
         content += "      </description>\n";
+        content += "         <description>Test</description>\n";
         content += "         <identifier>http://uts.cc.utexas.edu/~ailla/audio/sherzer/one_eyed_grandmother.ram</identifier>\n";
         content += "         <identifier>http://uts.cc.utexas.edu/~ailla/texts/sherzer/one_eyed_grandmother.pdf</identifier>\n";
@@ -516 +525 @@
 
         File cmdiFile = createCmdiFile("testOlac", content);
-        CMDIDigester digester = new CMDIDigester(getOlacFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getOlacFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("oai:ailla.utexas.edu:1", data.getId());
         List<String> resources = data.getResources();
@@ -532 +541 @@
         assertEquals("transcription", doc.getFieldValue("genre"));
         //  assertEquals("Kuna", doc.getFieldValue("subject"));
-        assertEquals(2, doc.getFieldValues("description").size());
+        Collection<Object> fieldValues = doc.getFieldValues("description");
+        assertEquals(3, fieldValues.size());
+        List<String> descriptions = new ArrayList(fieldValues);
+        Collections.sort(descriptions);
+        assertEquals("\n    Channel: Talking;\n    Genre: Traditional Narrative / Story;\n    Country: Panama;\n"
+                + "    Place of Recording: Mulatuppu;\n    Event: Community Gathering;\n"
+                + "    Institutional Affiliation: University of Texas at Austin;\n    Participant Information: Political Leader;\n"
+                + "      ", descriptions.get(0).toString());
+        assertEquals("Test", descriptions.get(1).toString());
+        assertEquals("The one-eyed grandmother is one of many traditional Kuna stories performed "
+                + "in the Kuna gathering house. This story, performed here by Pedro Arias, combines "
+                + "European derived motifs (Tom Thumb and Hansel and Gretel) with themes that seem more "
+                + "Kuna in origin. All are woven together and a moral is provided. Pedro Arias performed "
+                + "this story before a gathered audience in the morning..\n      ", descriptions.get(2).toString());
     }
 
@@ -568 +590 @@
 
         File cmdiFile = createCmdiFile("testOlac", content);
-        CMDIDigester digester = new CMDIDigester(getOlacFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getOlacFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("collection_ATILF_Resources.cmdi", data.getId());
         List<String> resources = data.getResources();
@@ -616 +638 @@
 
         File cmdiFile = createCmdiFile("testOlac", content);
-        CMDIDigester digester = new CMDIDigester(getLrtFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getLrtFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("clarin.eu:lrt:433", data.getId());
         List<String> resources = data.getResources();