Changeset 6356

Timestamp:

07/06/15 11:05:51 (9 years ago)

Author:

davor.ostojic@oeaw.ac.at

Message:

refers #768

default config path is set via system property: configFile

command line params are not processed by the script, just passed to java
there are two optional parameters:
-c <pathToConfigFile>
-l <pathToDataRoot> ... for processing only subset of data roots from configuration file

checkDataRoot method from MetadataImporter? won't exist if dataRoot doesn't exist and it returns the list of existing data roots

new method filterDataRoots:
if dataRoots are passed via -l option, importer will filter the list of existing with those from -l

logging

Location:

vlo/trunk/vlo-importer/src/main

Files:

• bin/vlo_solr_importer.sh (modified) (2 diffs)
• java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java (modified) (11 diffs)

vlo/trunk/vlo-importer/src/main/bin/vlo_solr_importer.sh

@@ -12 +12 @@
 fi
 
-# try to get the configuration file name from the command line options
-if [ 1 -eq $# ]; then
-CONFIG=$1
-else
-# fall back to default location
-echo No configuration location specified, using default
-CONFIG=${DIR}/"../config/VloConfig.xml"
-fi
+#default configuration file
+DFLT_CONFIG=${DIR}/"../config/VloConfig.xml"
 
 LOGDIR=${DIR}/../log/
@@ -35 +29 @@
 # to the JAVA command. Please note the a specification on the command line will
 # take preference over a specification as a property.
+# 
+# to process only subset of data roots from configuration file pass the list via command line:
+# -l path OR -l "path1 path2 ..."
+#
 
 $JAVA -Xmx3G \
     -cp "${DIR}:${DIR}/vlo-importer-${project.version}-importer.jar" \
+        -DconfigFile=${DFLT_CONFIG} \
     -DIMPORTER_LOG_DIR=${LOGDIR} \
-    eu.clarin.cmdi.vlo.importer.MetadataImporter -c "$CONFIG"
+    eu.clarin.cmdi.vlo.importer.MetadataImporter "$@"
 

vlo/trunk/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java

@@ -1 +1 @@
 package eu.clarin.cmdi.vlo.importer;
-
-import eu.clarin.cmdi.vlo.LanguageCodeUtils;
-import eu.clarin.cmdi.vlo.CommonUtils;
-import eu.clarin.cmdi.vlo.FacetConstants;
-import eu.clarin.cmdi.vlo.config.DataRoot;
-import eu.clarin.cmdi.vlo.config.VloConfig;
-import eu.clarin.cmdi.vlo.config.XmlVloConfigFactory;
 
 import java.io.File;
@@ -14 +7 @@
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -38 +33 @@
 import org.slf4j.LoggerFactory;
 
+import eu.clarin.cmdi.vlo.CommonUtils;
+import eu.clarin.cmdi.vlo.FacetConstants;
+import eu.clarin.cmdi.vlo.LanguageCodeUtils;
+import eu.clarin.cmdi.vlo.config.DataRoot;
+import eu.clarin.cmdi.vlo.config.VloConfig;
+import eu.clarin.cmdi.vlo.config.XmlVloConfigFactory;
+
 /**
  * The main metadataImporter class. Also contains the main function.
@@ -93 +95 @@
      */
     public MetadataImporter() {}
+    
+    public MetadataImporter(String clDatarootsList) {
+        this.clDatarootsList = clDatarootsList;
+    }
 
     /**
@@ -121 +127 @@
         initSolrServer();
         List<DataRoot> dataRoots = checkDataRoots();
+        
+        dataRoots = filterDataRootsWithCLArgs(dataRoots);
+        
         long start = System.currentTimeMillis();
         try {
@@ -202 +211 @@
     protected List<DataRoot> checkDataRoots() {
         List<DataRoot> dataRoots = config.getDataRoots();
+        List<DataRoot> existingDataRoots = new LinkedList<DataRoot>();
         for (DataRoot dataRoot : dataRoots) {
             if (!dataRoot.getRootFile().exists()) {
-                LOG.error("Root file " + dataRoot.getRootFile() + " does not exist. Probable configuration error so stopping import.");
-                System.exit(1);
-            }
-        }
-        return dataRoots;
+                LOG.warn("Root file " + dataRoot.getRootFile() + " does not exist. It could be configuration error! Proceeding with next ...");
+            } else{
+                existingDataRoots.add(dataRoot);
+            }
+            
+        }
+        return existingDataRoots;
+    }
+    
+    /**
+     * if user specified which data roots should be imported,
+     * list of existing data roots will be filtered with the list from user
+     * 
+     * @return
+     */
+    protected List<DataRoot> filterDataRootsWithCLArgs(List<DataRoot> dataRoots){
+        if(clDatarootsList == null)
+                return dataRoots;
+        
+        
+        LOG.info("Filtering configured data root files with command line arguments: \"" + clDatarootsList + "\"" ) ;
+        
+        LinkedList<File> fsDataRoots = new LinkedList<File>();
+        
+        List<String> paths = Arrays.asList((clDatarootsList.split("\\s+")));
+        
+        //Convert String paths to File objects for comparison
+        for(String path: paths)
+                fsDataRoots.add(new File(path));
+        
+        List<DataRoot> filteredDataRoots = new LinkedList<DataRoot>();
+        try{
+                //filter data
+        dr: for(DataRoot dataRoot: dataRoots){
+                        for(File fsDataRoot: fsDataRoots){
+                        if(fsDataRoot.getCanonicalPath().equals(dataRoot.getRootFile().getCanonicalPath())){
+                                filteredDataRoots.add(dataRoot);
+                                fsDataRoots.remove(fsDataRoot);
+                                continue dr;
+                        }
+                        }
+                        LOG.info("Root file " + dataRoot.getRootFile() + " will be omitted from processing");
+                }
+        }catch (IOException e){
+                filteredDataRoots = dataRoots;
+        }
+
+        
+                return filteredDataRoots;
     }
 
@@ -529 +583 @@
     public static LanguageCodeUtils languageCodeUtils;
     
+    //data roots passed from command line    
+    private String clDatarootsList = null;
+    
     /**
      * @param args
@@ -541 +598 @@
         // use the Apache cli framework for getting command line parameters
         Options options = new Options();
+        
+        // Data root list passed from command line with -l option
+        String cldrList = null;
 
         /**
          * Add a "c" option, the option indicating the specification of an XML
          * configuration file
+         * 
+         * "l" option - to specify which data roots (from config file) to import 
+         * imports all by default
          */
         options.addOption("c", true, "-c <file> : use parameters specified in <file>");
+        options.addOption("l", true, "-l <dataroot> [ ' ' <dataroot> ]* :  space separated list of dataroots to be processed.\n"
+                        + "If dataroot is not specified in config file it will be ignored.");
+        options.getOption("l").setOptionalArg(true);
         
         CommandLineParser parser = new PosixParser();
@@ -557 +623 @@
                 // the "c" option was specified, now get its value
                 configFile = cmd.getOptionValue("c");
+            }
+            
+            if(cmd.hasOption("l")){
+                cldrList = cmd.getOptionValue("l");
             }
             
@@ -599 +669 @@
             }
             System.out.println("Reading configuration from " + configUrl.toString());
+            LOG.info("Reading configuration from " + configUrl.toString());
             final XmlVloConfigFactory configFactory = new XmlVloConfigFactory(configUrl);
             MetadataImporter.config = configFactory.newConfig();
@@ -605 +676 @@
             // optionally, modify the configuration here
             // create and start the importer
-            MetadataImporter importer = new MetadataImporter();
+            MetadataImporter importer = new MetadataImporter(cldrList);
             importer.startImport();