Changeset 3175

Timestamp:

07/22/13 12:28:40 (11 years ago)

Author:

keeloo

Message:

Fixed some more errors

Location:

vlo/trunk/vlo_importer

Files:

• src/main/java/eu/clarin/cmdi/vlo/config/VloConfig.java (modified) (4 diffs)
• src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java (modified) (6 diffs)
• src/main/resources/VloConfig.xml (modified) (1 diff)
• vlo-importer.log.2013-07-18 (added)

vlo/trunk/vlo_importer/src/main/java/eu/clarin/cmdi/vlo/config/VloConfig.java

@@ -165 +165 @@
     
     @Element 
-    private static int maxDocsInSolrQueue = 0;
+    private static int minDocsInSolrQueue = 0;
     
     @Element 
@@ -292 +292 @@
     
     /**
-     * Get the value of the maxDocsSolrQueue parameter<br><br>
+     * Get the value of the minDocsInSolrQueue parameter<br><br>
      *
      * The value of the parameter indicates the number of documents in the
@@ -300 +300 @@
      * @return the value
      */
-    public static int getDocsSolrQueue (){
-        return maxDocsInSolrQueue;
-    }
-    
-    /**
-     * Set the value of the maxDocsSolrQueue parameter<br><br>
+    public static int getMinDocsInSolrQueue (){
+        return minDocsInSolrQueue;
+    }
+    
+    /**
+     * Set the value of the minDocsInSolrQueue parameter<br><br>
      *
      * The value of the parameter indicates the number of documents in the
@@ -313 +313 @@
      * @param param the value
      */
-    public static void setDocsSolrQueue (int param){
-        maxDocsInSolrQueue = param;
+    public static void setMinDocsInSolrQueue (int param){
+        minDocsInSolrQueue = param;
     }
     

vlo/trunk/vlo_importer/src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java

@@ -15 +15 @@
 import java.util.Map;
 import java.util.Set;
-import java.util.logging.Level;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -207 +206 @@
 
     /**
-     * Initialize SolrServer as specified in configuration file
+     * Create an interface to the SOLR server. 
+     * 
+     * After the interface has been created the importer can send documents to
+     * the server. Sending documents involves a queue. The importer adds
+     * documents to a queue, and dedicated threads will empty it, and
+     * effectively store store the documents.
      *
      * @throws MalformedURLException
@@ -214 +218 @@
         String solrUrl = VloConfig.getSolrUrl();
         LOG.info("Initializing Solr Server on " + solrUrl);
-        solrServer = new StreamingUpdateSolrServer(solrUrl, VloConfig.getMaxDocsInList(), 2) {
+        
+        /* Specify the number of documents in the queue that will trigger the
+         * threads, two of them, emptying it.
+         */
+        solrServer = new StreamingUpdateSolrServer(solrUrl, 
+                VloConfig.getMinDocsInSolrQueue(), 2) {
+                    /*
+                     * Let the super class method handle exceptions. Make the
+                     * exception available to the importer in the form of the
+                     * serverError variable.
+                     */
             @Override
-            public void handleError(Throwable ex) {
-                super.handleError(ex);
-                serverError = ex;
+            public void handleError(Throwable exception) {
+                super.handleError(exception);
+                serverError = exception;
             }
         };
@@ -361 +375 @@
     
     /**
-     * Send the list of documents prepared to the solr server
+     * Send the current list of documents to the SOLR server
      *
      * @throws SolrServerException
@@ -371 +385 @@
         
         LOG.info("Sending " + docs.size() + 
-                " docs to solr server queue. Total number of docs updated till now: " 
+                " docs to solr server queue. Number of docs updated till now: " 
                 + nrOFDocumentsUpdated);
 
         nrOFDocumentsUpdated += docs.size();
         // add the documents in the list to the solr queue
-        while (! done){
+        while (! done && wait <= VloConfig.getSolrTimeOut()){
             solrServer.add(docs); 
             done = (serverError == null) && (wait <= VloConfig.getSolrTimeOut());
             if (! done){
                 try {
-                    Thread.sleep (1000* wait);
-                } catch (InterruptedException ex) {
-                    java.util.logging.Logger.getLogger(
-                            MetadataImporter.class.getName()).log(Level.SEVERE, 
-                            null, ex);
+                    if (wait == 1) {
+                        LOG.info("Waiting 1 second for the solr server to respond");
+                    } else {
+                        LOG.info("Waiting ", wait,
+                                "seconds for the solr server to respond");
+                    }
+                    Thread.sleep(1000 * wait);
+                } catch (InterruptedException e) {
+                    LOG.info(e.toString());
                 }
                 wait = wait * 2;
             }
         }
-        
-        // turn wait into vlo parameter
         
         if (done) {
@@ -397 +413 @@
             docs = new ArrayList<SolrInputDocument>();
         } else {
+            // the documents haven't reached the queue
             if (wait > VloConfig.getSolrTimeOut()) {
-                // timeout
-                LOG.error("Timeout sending list of documents to solr queue");
+                LOG.error("Timeout sending list of documents to solr server queue");
             }
             throw new SolrServerException(serverError);

vlo/trunk/vlo_importer/src/main/resources/VloConfig.xml

@@ -5 +5 @@
     <maxDocsInList>128</maxDocsInList>
     
-    <maxDocsInSolrQueue>128</maxDocsInSolrQueue>
+    <minDocsInSolrQueue>128</minDocsInSolrQueue>
     
     <solrTimeOut>300</solrTimeOut>