Changeset 2826

Timestamp:

04/22/13 13:32:04 (11 years ago)

Author:

keeloo

Message:

Repaired the importer test: made it transparent regarding the
deleteAllFirst parameter.

Added the parameter and logic that causes large files to be
filtered. Added accompanying tests in the vlo test configuration
package.

Location:

vlo/trunk/vlo_importer/src

Files:

• main/java/eu/clarin/cmdi/vlo/config/VloConfig.java (modified) (2 diffs)
• main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java (modified) (10 diffs)
• main/resources/VloConfig.xml (modified) (1 diff)
• test/java/eu/clarin/cmdi/vlo/config/VloConfigTest.java (modified) (5 diffs)
• test/java/eu/clarin/cmdi/vlo/importer/MetadataImporterTest.java (modified) (4 diffs)

vlo/trunk/vlo_importer/src/main/java/eu/clarin/cmdi/vlo/config/VloConfig.java

@@ -162 +162 @@
     
     @Element 
-    private static int maxOnHeap = 1000;
+    private static int maxOnHeap = 0;
+    
+    @Element 
+    private static boolean useMaxFileSize = false;
+    
+    @Element 
+    private static int maxFileSize = 0;
     
     @Element
@@ -278 +284 @@
     
     /**
+     * Get the value of the useMaxFileSize parameter<br><br>
+     *
+     * For a description of the parameter, refer to the general VLO
+     * documentation.
+     *
+     * @return the value
+     */
+    public static boolean getUseMaxFileSize (){
+        return useMaxFileSize;
+    }
+    
+    /**
+     * Set the value of the useMaxFileSize parameter<br><br>
+     *
+     * For a description of the parameter, refer to the general VLO
+     * documentation.
+     *
+     * @param param the value
+     */
+    public static void setUseMaxFileSize (boolean param){
+        useMaxFileSize = param;
+    }
+    
+    /**
+     * Get the value of the maxFileSize parameter<br><br>
+     *
+     * For a description of the parameter, refer to the general VLO
+     * documentation.
+     *
+     * @return the value
+     */
+    public static int getMaxFileSize (){
+        return maxFileSize;
+    }
+    
+    /**
+     * Set the value of the maxFileSize parameter<br><br>
+     *
+     * For a description of the parameter, refer to the general VLO
+     * documentation.
+     *
+     * @param param the value
+     */
+    public static void setMaxFileSize (int param){
+        maxFileSize = param;
+    }
+    
+    /**
      * Get the value of the useHandleResolver parameter<br><br>
      *

vlo/trunk/vlo_importer/src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java

@@ -51 +51 @@
      * Log log log log
      */
-    private final static Logger LOG = LoggerFactory.getLogger(MetadataImporter.class);
+    protected final static Logger LOG = LoggerFactory.getLogger(MetadataImporter.class);
     /**
      * Some place to store errors.
@@ -88 +88 @@
      * Just to know what we have already done.
      */
-    private final Set<String> processedIds = new HashSet<String>();
+    protected final Set<String> processedIds = new HashSet<String>();
     /**
      * Some caching for solr documents (we are more efficient if we ram a whole
@@ -96 +96 @@
 
     // SOME STATS
-    private int nrOFDocumentsUpdated;
-    private int nrOfFilesAnalyzed = 0;
-    private int nrOfFilesWithoutId = 0;
-    private int nrOfFilesWithoutDataResources = 0;
-    private int nrOfFilesWithError = 0;
+    protected int nrOFDocumentsUpdated;
+    protected int nrOfFilesAnalyzed = 0;
+    protected int nrOfFilesWithoutId = 0;
+    protected int nrOfFilesWithoutDataResources = 0;
+    protected int nrOfFilesWithError = 0;
 
     /**
@@ -131 +131 @@
                 List<File> files = getFilesFromDataRoot(dataRoot.getRootFile());
                 for (File file : files) {
-                    LOG.debug("PROCESSING FILE: " + file.getAbsolutePath());
-                    processCmdi(file, dataRoot, processor);
+                    if (VloConfig.getUseMaxFileSize() && 
+                            file.length() > VloConfig.getMaxFileSize()) {
+                        LOG.info("Skipping " + file.getAbsolutePath() + " because it is too large.");
+                    } else {
+                        LOG.debug("PROCESSING FILE: " + file.getAbsolutePath());
+                        processCmdi(file, dataRoot, processor);
+                    }
                 }
                 if (!docs.isEmpty()) {
@@ -170 +175 @@
      * @return
      */
-    private List<DataRoot> checkDataRoots() {
+    protected List<DataRoot> checkDataRoots() {
         List<DataRoot> dataRoots = VloConfig.getDataRoots();
         for (DataRoot dataRoot : dataRoots) {
@@ -189 +194 @@
      * directory
      */
-    private List<File> getFilesFromDataRoot(File rootFile) {
+    protected List<File> getFilesFromDataRoot(File rootFile) {
         List<File> result = new ArrayList<File>();
         if (rootFile.isFile()) {
@@ -226 +231 @@
      * @throws IOException
      */
-    private void processCmdi(File file, DataRoot dataOrigin, CMDIDataProcessor processor) throws SolrServerException, IOException {
+    protected void processCmdi(File file, DataRoot dataOrigin, CMDIDataProcessor processor) throws SolrServerException, IOException {
         nrOfFilesAnalyzed++;
         CMDIData cmdiData = null;
@@ -262 +267 @@
      * @return true if id is acceptable, false otherwise
      */
-    private boolean idOk(String id) {
+    protected boolean idOk(String id) {
         return id != null && !id.isEmpty();
     }
@@ -278 +283 @@
      * @throws IOException
      */
-    private void updateDocument(SolrInputDocument solrDocument, CMDIData cmdiData, File file, DataRoot dataOrigin) throws SolrServerException,
+    protected void updateDocument(SolrInputDocument solrDocument, CMDIData cmdiData, File file, DataRoot dataOrigin) throws SolrServerException,
             IOException {
         if (!solrDocument.containsKey(FacetConstants.FIELD_COLLECTION)) {
@@ -318 +323 @@
      * a type is overwritten and already in the solrDocument we take that type.
      */
-    private void addResourceData(SolrInputDocument solrDocument, CMDIData cmdiData) {
+    protected void addResourceData(SolrInputDocument solrDocument, CMDIData cmdiData) {
         List<Object> fieldValues = solrDocument.containsKey(FacetConstants.FIELD_RESOURCE_TYPE) ? new ArrayList<Object>(solrDocument
                 .getFieldValues(FacetConstants.FIELD_RESOURCE_TYPE)) : null;

vlo/trunk/vlo_importer/src/main/resources/VloConfig.xml

@@ -4 +4 @@
     
     <maxOnHeap>128</maxOnHeap>
+    
+    <useMaxFileSize>false</useMaxFileSize>
+    
+    <maxFileSize>20000000</maxFileSize>
     
     <useHandleResolver>false</useHandleResolver>

vlo/trunk/vlo_importer/src/test/java/eu/clarin/cmdi/vlo/config/VloConfigTest.java

@@ -88 +88 @@
 
     /**
-     * Test the getVloHomeLink method
+     * Test the getMaxOnHeap method
      */
     @Test
@@ -102 +102 @@
 
     /**
-     * Test the setVloHomeLink method
+     * Test the setMaxOnHeap method
      */
     @Test
@@ -119 +119 @@
     
     /**
+     * Test the getUseMaxFileSize method
+     */
+    @Test
+    public void testGetUseMaxFileSize() {
+        
+        System.out.println("getUseMaxFileSize");
+        
+        boolean expResult = false;
+        boolean result = VloConfig.getUseMaxFileSize();
+        
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test the setSetUseMaxFileSize method
+     */
+    @Test
+    public void testSetUseMaxFileSize() {
+        
+        System.out.println("setUseMaxFileSize");
+        
+        boolean param = true;
+        
+        VloConfig.setUseMaxFileSize(param);
+
+        boolean result = VloConfig.getUseMaxFileSize();
+        
+        assertEquals(param, result);
+    }
+    
+        /**
+     * Test the getMaxFileSize method
+     */
+    @Test
+    public void testGetMaxFileSize() {
+        
+        System.out.println("getMaxFileSize");
+        
+        int expResult = 20000000;
+        int result = VloConfig.getMaxFileSize();
+        
+        assertEquals(expResult, result);
+    }
+
+    /**
+     * Test the setMaxFileSize method
+     */
+    @Test
+    public void testSetMaxFileSize() {
+        
+        System.out.println("setMaxFileSize");
+        
+        int param = 10000000;
+        
+        VloConfig.setMaxFileSize(param);
+
+        int result = VloConfig.getMaxFileSize();
+        
+        assertEquals(param, result);
+    }
+    
+    /**
      * Test the getHandleResolver method
      */
@@ -124 +186 @@
     public void testGetUseHandleResolver() {
         
-        System.out.println("getMaxOnHeap");
+        System.out.println("getUseHandleResolver");
         
         boolean expResult = false;
@@ -138 +200 @@
     public void testSetUseHandleResolver() {
         
-        System.out.println("setMaxOnHeap");
+        System.out.println("setUseHandleResolver");
         
         boolean param = true;

vlo/trunk/vlo_importer/src/test/java/eu/clarin/cmdi/vlo/importer/MetadataImporterTest.java

@@ -11 +11 @@
 import java.util.Collections;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.common.SolrInputDocument;
@@ -236 +238 @@
         final List<SolrInputDocument> result = new ArrayList<SolrInputDocument>();
                 
-        // read configuration in ImporterTestCase.setup; change it now
+        /*
+         * Read configuration in ImporterTestCase.setup and change the setup to
+         * suit the test.
+         */
         
         modifyConfig(rootFile);
@@ -242 +247 @@
         MetadataImporter importer;
         importer = new MetadataImporter() {
+            /*
+             * Because in the test, the solr server is not assumed to be 
+             * available, override the importer's class startImport method by
+             * leaving out interaction with server. 
+             * 
+             * By invoking the processCmdi method, the class being defined here
+             * needs to anticipate on an exception possibly thrown by the 
+             * processCmdi method invoking the sendDocs method. Please note 
+             * however, that the latter method is overriden, and the actual 
+             * database is being replaced by an array of documents.
+             */
             @Override
-            protected void initSolrServer() throws MalformedURLException {
-                //do nothing no solrserver in test
+            void startImport() throws MalformedURLException {
+
+                List<DataRoot> dataRoots = checkDataRoots();
+                long start = System.currentTimeMillis();
+                try {
+
+                    for (DataRoot dataRoot : dataRoots) {
+                        LOG.info("Start of processing: " + 
+                                dataRoot.getOriginName());
+                        CMDIDataProcessor processor = new 
+                                CMDIParserVTDXML(POST_PROCESSORS);
+                        List<File> files = 
+                                getFilesFromDataRoot(dataRoot.getRootFile());
+                        for (File file : files) {
+                            if (VloConfig.getUseMaxFileSize()
+                                    && file.length() > 
+                                    VloConfig.getMaxFileSize()) {
+                                LOG.info("Skipping " + file.getAbsolutePath() + 
+                                        " because it is too large.");
+                            } else {
+                                LOG.debug("PROCESSING FILE: " + 
+                                        file.getAbsolutePath());                
+                                /*
+                                 * Anticipate on the solr exception that will
+                                 * never by raised because sendDocs is overriden
+                                 * in a suitable way.
+                                 */
+                                try {
+                                    processCmdi(file, dataRoot, processor);
+                                } catch (SolrServerException ex) {
+                                    Logger.getLogger(MetadataImporterTest.class.getName()).log(Level.SEVERE, null, ex);
+                                }
+                            }
+                        }
+                        if (!docs.isEmpty()) {
+                            sendDocs();
+                        }
+                        LOG.info("End of processing: " + 
+                                dataRoot.getOriginName());
+                    }
+                    
+                } catch (IOException e) {
+                    LOG.error("error updating files:\n", e);
+                } finally {
+
+                }
+                long took = (System.currentTimeMillis() - start) / 1000;
+                LOG.info("Found " + nrOfFilesWithoutId + 
+                        " file(s) without an id. (id is generated based on fileName but that may not be unique)");
+                LOG.info("Found " + nrOfFilesWithError + 
+                        " file(s) with errors.");
+                LOG.info("Found " + nrOfFilesWithoutDataResources
+                        + " file(s) without data resources (metadata descriptions without resources are ignored).");
+                LOG.info("Update of " + nrOFDocumentsUpdated + " took " + took + 
+                        " secs. Total nr of files analyzed " + nrOfFilesAnalyzed);
             }
 
+            /*
+             * Replace the server's database by a document array
+             */
             @Override
-            protected void sendDocs() throws SolrServerException, IOException {
-            //overriding here so we can test the docs
+            protected void sendDocs() throws IOException {
+                
                 result.addAll(this.docs);
                 docs = new ArrayList<SolrInputDocument>();
@@ -266 +338 @@
         dataRoot.setPrefix("http://example.com");
         VloConfig.setDataRoots(Collections.singletonList(dataRoot));
-        
-        /**
-         * Please observe that if the deleteAllFirst parameter is true, the
-         * startImport method in the MetaDataImporter class will refer to an
-         * instance of the solr server. Because in this test initSolrServer
-         * method is empty, no solr server is create. Therefore, the value of
-         * the deleteAllFirst parameter needs to be false.
-         */
-        VloConfig.setDeleteAllFirst(false);
     }