Changeset 996 for vlo/trunk/vlo_webapp/src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessorTest.java

Timestamp:

12/15/10 12:48:10 (13 years ago)

Author:

patdui

Message:

• Added vtd-xml parser which is 10x faster (5mins vs. 1hour for 200.000 cmdi files) then DOM parser. Still supports xpath to query the data out.
• Made it possible to delete only part of the index, you can delete a whole origin.

File:

• vlo/trunk/vlo_webapp/src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessorTest.java (moved) (moved from vlo/trunk/vlo_webapp/src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDigesterTest.java) (10 diffs)

vlo/trunk/vlo_webapp/src/test/java/eu/clarin/cmdi/vlo/importer/CMDIDataProcessorTest.java

@@ -7 +7 @@
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
@@ -17 +20 @@
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
-public class CMDIDigesterTest {
+public class CMDIDataProcessorTest {
 
     private static File testDir;
+
+    private CMDIDataProcessor getDataParser(FacetMapping map) {
+        return new CMDIParserVTDXML(map);
+        //        return new CMDIDigester(map);
+    }
 
     @Test
@@ -66 +74 @@
         content += "</CMD>\n";
         File cmdiFile = createCmdiFile("testCorpus", content);
-        CMDIDigester digester = new CMDIDigester(getIMDIFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getIMDIFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("test-hdl:1839/00-0000-0000-0000-0001-D", data.getId());
         List<String> resources = data.getResources();
@@ -362 +370 @@
         content += "</CMD>\n";
         File cmdiFile = createCmdiFile("testSession", content);
-        CMDIDigester digester = new CMDIDigester(getIMDIFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getIMDIFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("test-hdl:1839/00-0000-0000-0009-294C-9", data.getId());
         List<String> resources = data.getResources();
@@ -452 +460 @@
         content += "</CMD>\n";
         File cmdiFile = createCmdiFile("testSession", content);
-        CMDIDigester digester = new CMDIDigester(getIMDIFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getIMDIFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("test-hdl:1839/00-0000-0000-0009-294C-9", data.getId());
         List<String> resources = data.getResources();
@@ -505 +513 @@
         content += "         <description>The one-eyed grandmother is one of many traditional Kuna stories performed in the Kuna gathering house. This story, performed here by Pedro Arias, combines European derived motifs (Tom Thumb and Hansel and Gretel) with themes that seem more Kuna in origin. All are woven together and a moral is provided. Pedro Arias performed this story before a gathered audience in the morning..\n";
         content += "      </description>\n";
+        content += "         <description>Test</description>\n";
         content += "         <identifier>http://uts.cc.utexas.edu/~ailla/audio/sherzer/one_eyed_grandmother.ram</identifier>\n";
         content += "         <identifier>http://uts.cc.utexas.edu/~ailla/texts/sherzer/one_eyed_grandmother.pdf</identifier>\n";
@@ -516 +525 @@
 
         File cmdiFile = createCmdiFile("testOlac", content);
-        CMDIDigester digester = new CMDIDigester(getOlacFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getOlacFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("oai:ailla.utexas.edu:1", data.getId());
         List<String> resources = data.getResources();
@@ -532 +541 @@
         assertEquals("transcription", doc.getFieldValue("genre"));
         //  assertEquals("Kuna", doc.getFieldValue("subject"));
-        assertEquals(2, doc.getFieldValues("description").size());
+        Collection<Object> fieldValues = doc.getFieldValues("description");
+        assertEquals(3, fieldValues.size());
+        List<String> descriptions = new ArrayList(fieldValues);
+        Collections.sort(descriptions);
+        assertEquals("\n    Channel: Talking;\n    Genre: Traditional Narrative / Story;\n    Country: Panama;\n"
+                + "    Place of Recording: Mulatuppu;\n    Event: Community Gathering;\n"
+                + "    Institutional Affiliation: University of Texas at Austin;\n    Participant Information: Political Leader;\n"
+                + "      ", descriptions.get(0).toString());
+        assertEquals("Test", descriptions.get(1).toString());
+        assertEquals("The one-eyed grandmother is one of many traditional Kuna stories performed "
+                + "in the Kuna gathering house. This story, performed here by Pedro Arias, combines "
+                + "European derived motifs (Tom Thumb and Hansel and Gretel) with themes that seem more "
+                + "Kuna in origin. All are woven together and a moral is provided. Pedro Arias performed "
+                + "this story before a gathered audience in the morning..\n      ", descriptions.get(2).toString());
     }
 
@@ -568 +590 @@
 
         File cmdiFile = createCmdiFile("testOlac", content);
-        CMDIDigester digester = new CMDIDigester(getOlacFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getOlacFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("collection_ATILF_Resources.cmdi", data.getId());
         List<String> resources = data.getResources();
@@ -616 +638 @@
 
         File cmdiFile = createCmdiFile("testOlac", content);
-        CMDIDigester digester = new CMDIDigester(getLrtFacetMap());
-        CMDIData data = digester.process(cmdiFile);
+        CMDIDataProcessor processor = getDataParser(getLrtFacetMap());
+        CMDIData data = processor.process(cmdiFile);
         assertEquals("clarin.eu:lrt:433", data.getId());
         List<String> resources = data.getResources();