Changeset 4747

Timestamp:

03/18/14 15:26:03 (11 years ago)

Author:

Twan Goosen

Message:

accept relative file path as config location in importer

File:

• vlo/branches/vlo-3.0/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java (modified) (23 diffs)

vlo/branches/vlo-3.0/vlo-importer/src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java

@@ -35 +35 @@
 import org.slf4j.LoggerFactory;
 
-
 /**
  * The main metadataImporter class. Also contains the main function.
@@ -44 +43 @@
  * and so on.
  */
-
 @SuppressWarnings({"serial"})
 public class MetadataImporter {
 
     /**
-     * Defines which files to try and parse.
-     * In this case all files ending in "xml" or "cmdi".
-     */
-    private static final String[] VALID_CMDI_EXTENSIONS = new String[] { "xml", "cmdi" };
+     * Defines which files to try and parse. In this case all files ending in
+     * "xml" or "cmdi".
+     */
+    private static final String[] VALID_CMDI_EXTENSIONS = new String[]{"xml", "cmdi"};
 
     /**
@@ -73 +71 @@
      */
     final static Map<String, PostProcessor> POST_PROCESSORS = new HashMap<String, PostProcessor>();
+
     static {
         POST_PROCESSORS.put(FacetConstants.FIELD_ID, new IdPostProcessor());
-                POST_PROCESSORS.put(FacetConstants.FIELD_CONTINENT, new ContinentNamePostProcessor());
+        POST_PROCESSORS.put(FacetConstants.FIELD_CONTINENT, new ContinentNamePostProcessor());
         POST_PROCESSORS.put(FacetConstants.FIELD_COUNTRY, new CountryNamePostProcessor());
         POST_PROCESSORS.put(FacetConstants.FIELD_LANGUAGE, new LanguageCodePostProcessor());
@@ -83 +82 @@
         POST_PROCESSORS.put(FacetConstants.FIELD_CLARIN_PROFILE, new CMDIComponentProfileNamePostProcessor());
     }
-    
+
     /**
      * Constructor
-     * 
+     *
      * @param
      */
-    public MetadataImporter (){
-    }
-
-    /**
-     * Contains MDSelflinks (usually).
-     * Just to know what we have already done.
+    public MetadataImporter() {
+    }
+
+    /**
+     * Contains MDSelflinks (usually). Just to know what we have already done.
      */
     protected final Set<String> processedIds = new HashSet<String>();
@@ -129 +127 @@
                 LOG.info("Deleting original data done.");
             }
-            
+
             // Import the specified data roots
             for (DataRoot dataRoot : dataRoots) {
@@ -141 +139 @@
                 List<File> files = getFilesFromDataRoot(dataRoot.getRootFile());
                 for (File file : files) {
-                    if (config.getMaxFileSize() > 0 && 
-                            file.length() > config.getMaxFileSize()) {
+                    if (config.getMaxFileSize() > 0
+                            && file.length() > config.getMaxFileSize()) {
                         LOG.info("Skipping " + file.getAbsolutePath() + " because it is too large.");
                     } else {
@@ -154 +152 @@
                 LOG.info("End of processing: " + dataRoot.getOriginName());
             }
-            
+
             // delete outdated entries (based on maxDaysInSolr parameter)
-            if(config.getMaxDaysInSolr() > 0 && config.getDeleteAllFirst() == false) {
-                LOG.info("Deleting old files that were not seen for more than "+config.getMaxDaysInSolr()+" days...");
-                solrServer.deleteByQuery(FacetConstants.FIELD_LAST_SEEN+":[* TO NOW-"+config.getMaxDaysInSolr()+"DAYS]");
+            if (config.getMaxDaysInSolr() > 0 && config.getDeleteAllFirst() == false) {
+                LOG.info("Deleting old files that were not seen for more than " + config.getMaxDaysInSolr() + " days...");
+                solrServer.deleteByQuery(FacetConstants.FIELD_LAST_SEEN + ":[* TO NOW-" + config.getMaxDaysInSolr() + "DAYS]");
                 LOG.info("Deleting old files done.");
             }
@@ -171 +169 @@
                     solrServer.commit();
                     buildSuggesterIndex();
-                }                
+                }
             } catch (SolrServerException e) {
                 LOG.error("cannot commit:\n", e);
@@ -223 +221 @@
 
     /**
-     * Create an interface to the SOLR server. 
-     * 
+     * Create an interface to the SOLR server.
+     *
      * After the interface has been created the importer can send documents to
      * the server. Sending documents involves a queue. The importer adds
@@ -235 +233 @@
         String solrUrl = config.getSolrUrl();
         LOG.info("Initializing Solr Server on " + solrUrl);
-        
+
         /* Specify the number of documents in the queue that will trigger the
          * threads, two of them, emptying it.
          */
-        solrServer = new ConcurrentUpdateSolrServer(solrUrl, 
+        solrServer = new ConcurrentUpdateSolrServer(solrUrl,
                 config.getMinDocsInSolrQueue(), 2) {
                     /*
@@ -246 +244 @@
                      * serverError variable.
                      */
-            @Override
-            public void handleError(Throwable exception) {
-                super.handleError(exception);
-                serverError = exception;
-            }
-        };
+                    @Override
+                    public void handleError(Throwable exception) {
+                        super.handleError(exception);
+                        serverError = exception;
+                    }
+                };
     }
 
@@ -280 +278 @@
             if (solrDocument != null) {
                 if (!cmdiData.getDataResources().isEmpty() || !cmdiData.getLandingPageResources().isEmpty()
-                                || !cmdiData.getSearchResources().isEmpty() || !cmdiData.getSearchPageResources().isEmpty()
-                                || cmdiData.getMetadataResources().isEmpty() ) {
+                        || !cmdiData.getSearchResources().isEmpty() || !cmdiData.getSearchPageResources().isEmpty()
+                        || cmdiData.getMetadataResources().isEmpty()) {
                     // We only add metadata files that have
                     //  1) data resources or
-                        //      2) a landing page or
-                        //      3) a search service (like SRU/CQL) or
-                        //      4) a search page or 
+                    //  2) a landing page or
+                    //  3) a search service (like SRU/CQL) or
+                    //  4) a search page or 
                     //  5) that have none of the above but also lack any metadata links (e.g. olac files describing a corpus with a link to the original archive).
                     // Other files will have only metadata resources and are considered 'collection' metadata files they
@@ -343 +341 @@
             solrDocument.addField(FacetConstants.FIELD_LANDINGPAGE, resource.getResourceName());
         }
-        
+
         // add search page resource
         for (Resource resource : cmdiData.getSearchPageResources()) {
             solrDocument.addField(FacetConstants.FIELD_SEARCHPAGE, resource.getResourceName());
         }
-        
+
         // add timestamp
         Date dt = new Date();
         SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
         solrDocument.addField(FacetConstants.FIELD_LAST_SEEN, df.format(dt));
-        
+
         // add resource proxys      
         addResourceData(solrDocument, cmdiData);
@@ -367 +365 @@
      * overwritten by some more specific xpath (as in the LRT cmdi files). So if
      * a type is overwritten and already in the solrDocument we take that type.
-     * 
-     * TODO evaluate odd connection between FIELD_FORMAT and ResourceProxy-Mimetypes
+     *
+     * TODO evaluate odd connection between FIELD_FORMAT and
+     * ResourceProxy-Mimetypes
      */
     protected void addResourceData(SolrInputDocument solrDocument, CMDIData cmdiData) {
@@ -390 +389 @@
                 format = CommonUtils.normalizeMimeType(mimeType);
             }
-            
+
             FormatPostProcessor processor = new FormatPostProcessor();
             mimeType = processor.process(mimeType);
-            
+
             // TODO check should probably be moved into Solr (by using some minimum length filter)
-            if(!mimeType.equals(""))
-                solrDocument.addField(FacetConstants.FIELD_FORMAT, mimeType);
+            if (!mimeType.equals("")) {
+                solrDocument.addField(FacetConstants.FIELD_FORMAT, mimeType);
+            }
             solrDocument.addField(FacetConstants.FIELD_RESOURCE, mimeType + FacetConstants.FIELD_RESOURCE_SPLIT_CHAR
                     + resource.getResourceName());
         }
     }
-    
+
     /**
      * Send current list of SolrImputDocuments to SolrServer and clears list
@@ -418 +418 @@
         docs = new ArrayList<SolrInputDocument>();
     }
-    
+
     /**
      * Builds suggester index for autocompletion
@@ -426 +426 @@
      */
     private void buildSuggesterIndex() throws SolrServerException, MalformedURLException {
-        LOG.info("Building index for autocompletion.");
-        HashMap<String,String> paramMap = new HashMap<String, String>();
-        paramMap.put("qt", "/suggest");
-        paramMap.put("spellcheck.build", "true");
-        SolrParams params = new MapSolrParams(paramMap);
-        solrServer.query(params);
+        LOG.info("Building index for autocompletion.");
+        HashMap<String, String> paramMap = new HashMap<String, String>();
+        paramMap.put("qt", "/suggest");
+        paramMap.put("spellcheck.build", "true");
+        SolrParams params = new MapSolrParams(paramMap);
+        solrServer.query(params);
     }
 
     public static VloConfig config;
-    
+
     /**
      * @param args
      * @throws IOException
      */
-    public static void main(String[] args) throws MalformedURLException, IOException { 
-
-        
+    public static void main(String[] args) throws MalformedURLException, IOException {
+
         // path to the configuration file
         String configFile = null;
-        
+
         // use the Apache cli framework for getting command line parameters
         Options options = new Options();
@@ -461 +460 @@
             CommandLine cmd = parser.parse(options, args);
             if (cmd.hasOption("c")) {
-                
+
                 // the "c" option was specified, now get its value
                 configFile = cmd.getOptionValue("c");
@@ -467 +466 @@
 
         } catch (org.apache.commons.cli.ParseException ex) {
-            
+
             /**
              * Caught an exception caused by command line parsing. Try to get
@@ -473 +472 @@
              * property.
              */
-
             String message = "Command line parsing failed. " + ex.getMessage();
             LOG.error(message);
             System.err.println(message);
         }
-        
-        if (configFile == null){
+
+        if (configFile == null) {
 
             String message;
@@ -485 +483 @@
             message = "Could not get config file name via the command line, trying the system properties.";
             LOG.info(message);
-            
+
             String key;
 
@@ -493 +491 @@
 
         if (configFile == null) {
-            
+
             String message;
-            
+
             message = "Could not get filename as system property either - stopping.";
             LOG.error(message);
         } else {
             // read the configuration from the externally supplied file
-            XmlVloConfigFactory configFactory = new XmlVloConfigFactory(new URL(configFile));
+            final URL configUrl;
+            if (configFile.startsWith("file:")) {
+                configUrl = new URL(configFile);
+            } else {
+                configUrl = new File(configFile).toURI().toURL();
+            }
+            System.out.println("Reading configuration from " + configUrl.toString());
+            final XmlVloConfigFactory configFactory = new XmlVloConfigFactory(configUrl);
             MetadataImporter.config = configFactory.newConfig();
 
             // optionally, modify the configuration here
-
             // create and start the importer
             MetadataImporter importer = new MetadataImporter();
@@ -510 +514 @@
 
             // finished importing
-
             if (MetadataImporter.config.printMapping()) {
                 File file = new File("xsdMapping.txt");