Changeset 2604


Ignore:
Timestamp:
02/21/13 14:40:39 (11 years ago)
Author:
keeloo
Message:

Added VloConfig? to the importer. Routed importer tests via via the VloConfig? class.

Location:
vlo/branches/vlo-2.13-param/vlo_webapp/src
Files:
4 deleted
6 edited

Legend:

Unmodified
Added
Removed

  • vlo/branches/vlo-2.13-param/vlo_webapp/src/main/java/eu/clarin/cmdi/vlo/VloApplication.java

    r2600 r2604  
    2020    private final SearchResultsDao searchResults;
    2121
    22     // application configuration object
     22    // application configuration
    2323    static VloConfig config;
    2424   
    … …  
    4040           
    4141            /**
    42              * Send the servlet context to the configuration object to enable it
    43              * to read an externel VloConfig.xml configuration file.
     42             * Send the application context to the configuration object to
     43             * enable it to read an external {@literal VloConfig.xml}
     44             * configuration file.
    4445             */
    4546           
    4647            config = VloConfig.switchToExternalConfig(servletContext);
    47 
    48             /**
    49              * Instead of obtaining the SolrUrl parameter from the context, read
    50              * the name of another instance - preferably the one that the
    51              * importer uses for its configuration - from the context. Once this
    52              * has been done, the whole web application configuration could be
    53              * refreshed. Note: add a method for refreshing the current
    54              * configuration to the VloConfig class.
    55              */
    5648        }
    5749    }

  • vlo/branches/vlo-2.13-param/vlo_webapp/src/main/java/eu/clarin/cmdi/vlo/config/VloConfig.java

    r2600 r2604  
    33import java.io.UnsupportedEncodingException;
    44import java.net.URLEncoder;
     5import java.util.List;
    56import javax.servlet.ServletContext;
    67import org.simpleframework.xml.Element;
    78import org.simpleframework.xml.ElementArray;
     9import org.simpleframework.xml.ElementList;
    810import org.simpleframework.xml.Root;
    911import org.slf4j.LoggerFactory;
    … …  
    3638 * in the WebAppConfig class. When the application invokes Simple by<br><br>
    3739 *
    38  * WebAppConfig.open();<br><br>
     40 * WebAppConfig.get();<br><br>
    3941 *
    4042 * the parameter itself is accessed by<br><br>
    4143 *
    42  * WebAppConfig.open().getParameterMember();<br><br>
     44 * WebAppConfig.get().getParameterMember();<br><br>
    4345 *
    4446 * If you want to add a type of member that is not included in the class yet,
    … …  
    117119     * WebAppConfig.getParameterMember()<br><br>
    118120     *
    119      * for example, would not be valid. On encountering open() however, a new
     121     * for example, would not be valid. On encountering get() however, a new
    120122     * static context is opened, and from that, getParameterMember() can be
    121123     * invoked:<br><br>
    122124     *
    123      * WebAppConfig.open().getParameterMember()<br><br>
     125     * WebAppConfig.get().getParameterMember()<br><br>
    124126     *
    125127     * @param fileName
    … …  
    174176
    175177    /**
    176      * Web application parameter members<br><br>
     178     * VLO application parameter members<br><br>
    177179     *
    178180     * Initialize a member corresponding to application parameters with an empty
    … …  
    183185     * member parameters.
    184186     */
    185     @Element // annotation directive as defined by Simple
     187   
     188    /**
     189     * Flag to signal the records in the data to be deleted before the ingestion
     190     * starts.
     191     */
     192    @Element // directive for Simple
     193    private boolean deleteAllFirst = false;
     194   
     195    /**
     196     * Flag that leads to the printing of XPATH mappings encountered. Note: need
     197     * to be more specific on this.
     198     */
     199    @Element
     200    private boolean printMapping = false;
     201   
     202    /**
     203     * A list of data roots, that is: directories from which the importer
     204     * collects meta data. Note: need to elaborate on this.
     205     */
     206    @ElementList // directive for Simple
     207    private List<DataRoot> dataRoots;
     208   
     209    public List<DataRoot> getDataRoots() {
     210        return dataRoots;
     211    }
     212   
     213    @Element
    186214    private String vloHomeLink = "";
    187215   
    … …  
    216244     * description of an array of three facet fields<br><br>
    217245     *
    218      * {@literal <facetFields length="3">}<br> {@literal    <facetField>}<br>
    219      * {@literal       fieldOne}<br> {@literal    </facetField>}<br>
    220      * {@literal    <facetField>}<br> {@literal       fieldTwo}<br>
    221      * {@literal    </facetField>}<br> {@literal    <facetField>}<br>
    222      * {@literal       fieldThree}<br> {@literal    </facetField>}<br>
     246     * {@literal <facetFields length="3">}<br>
     247     * {@literal    <facetField>}<br>
     248     * {@literal       fieldOne}<br>
     249     * {@literal    </facetField>}<br>
     250     * {@literal    <facetField>}<br>
     251     * {@literal       fieldTwo}<br>
     252     * {@literal    </facetField>}<br>
     253     * {@literal    <facetField>}<br>
     254     * {@literal       fieldThree}<br>
     255     * {@literal    </facetField>}<br>
    223256     * {@literal </facetFields>}<br><br>
    224257     *
    225      * To let Spring now it has to interpret the facetFields element as an
    226      * array, use the ElementArray directive. Use the directive to let Spring
     258     * To let Simple now it has to interpret the facetFields element as an
     259     * array, use the ElementArray directive. Use the directive to let Simple
    227260     * know that the elements inside 'facetFields' are named 'facetField'.
    228261     */
    … …  
    254287   
    255288    /**
    256      * Get the VloHomeLink parameter<br><br>
    257      *
    258      * For a description of the parameter, refer to the general VLO
    259      * documentation.
    260      *
    261      * @return the parameter
     289     * Set the value of the dataRoots parameter<br><br>
     290     *
     291     * For a description of the parameter, refer to the general VLO
     292     * documentation.
     293     *
     294     * @param the value
     295     */
     296    public void setDataRoots(List<DataRoot> dataRoots) {
     297        this.dataRoots = dataRoots;
     298    }
     299
     300    /**
     301     * Set the value deleteAllFirst parameter<br><br>
     302     *
     303     * For a description of the parameter, refer to the general VLO
     304     * documentation.
     305     *
     306     * @return the value
     307     */
     308    public void setDeleteAllFirst(boolean deleteAllFirst) {
     309        this.deleteAllFirst = deleteAllFirst;
     310    }
     311
     312    /**
     313     * Get the value of the deleteAllFirst parameter<br><br>
     314     *
     315     * For a description of the parameter, refer to the general VLO
     316     * documentation.
     317     *
     318     * @return the value
     319     */
     320    public boolean isDeleteAllFirst() {
     321        return deleteAllFirst;
     322    }
     323
     324    /**
     325     * Set the value of the printMapping parameter<br><br>
     326     *
     327     * For a description of the parameter, refer to the general VLO
     328     * documentation.
     329     *
     330     * @param the value
     331     */
     332    public void setPrintMapping(boolean printMapping) {
     333        this.printMapping = printMapping;
     334    }
     335
     336    /**
     337     * Get the value of the printMapping parameter<br><br>
     338     *
     339     * For a description of the parameter, refer to the general VLO
     340     * documentation.
     341     *
     342     * @return the value
     343     */
     344    public boolean isPrintMapping() {
     345        return printMapping;
     346    }
     347   
     348    /**
     349     * Get the value of the VloHomeLink parameter<br><br>
     350     *
     351     * For a description of the parameter, refer to the general VLO
     352     * documentation.
     353     *
     354     * @return the value
    262355     */
    263356    public String getVloHomeLink() {
    … …  
    266359
    267360    /**
    268      * Set the VloHomeLink parameter<br><br>
    269      *
    270      * For a description of the parameter, refer to the general VLO
    271      * documentation.
    272      *
    273      * @param vloHomeLink the parameter
     361     * Set the value of the VloHomeLink parameter<br><br>
     362     *
     363     * For a description of the parameter, refer to the general VLO
     364     * documentation.
     365     *
     366     * @param vloHomeLink the value
    274367     */
    275368    public void setVloHomeLink(String link) {
    … …  
    278371
    279372    /**
    280      * Get the SolrUrl parameter<br><br>
    281      *
    282      * For a description of the parameter, refer to the general VLO
    283      * documentation.
    284      *
    285      * @param solrUrl the parameter
     373     * Get the value of the SolrUrl parameter<br><br>
     374     *
     375     * For a description of the parameter, refer to the general VLO
     376     * documentation.
     377     *
     378     * @param solrUrl the value
    286379     */
    287380    public String getSolrUrl() {
    … …  
    290383
    291384    /**
    292      * Get the ProfileSchemaUrl by profileId parameter<br><br>
     385     * Get the value of the ProfileSchemaUrl by profileId parameter<br><br>
    293386     *
    294387     * For a description of the schema, refer to the general VLO documentation.
    295388     * Note: the profileId needs to be expanded.
    296389     *
    297      * @return the parameter
     390     * @return the value
    298391     */
    299392    public String getComponentRegistryProfileSchema(String id) {
    … …  
    302395
    303396    /**
    304      * Set the ProfileSchemaUrl parameter<br><br>
     397     * Set the value of the ProfileSchemaUrl parameter<br><br>
    305398     *
    306399     * For a description of the schema, refer to the general VLO documentation.
    307400     * Note: the profileId needs to be expanded.
    308401     *
    309      * @param profileId the parameter
     402     * @param profileId the value
    310403     */
    311404    public void setProfileSchemaUrl(String url) {
    … …  
    314407
    315408    /**
    316      * Get the ComponentRegisteryRESTURL parameter<br><br>
    317      *
    318      * For a description of the parameter, refer to the general VLO
    319      * documentation.
    320      *
    321      * @return the parameter
     409     * Get the value of the ComponentRegisteryRESTURL parameter<br><br>
     410     *
     411     * For a description of the parameter, refer to the general VLO
     412     * documentation.
     413     *
     414     * @return the value
    322415     */
    323416    public String getComponentRegistryRESTURL() {
    … …  
    326419
    327420    /**
    328      * Set the ComponentRegisteryRESTURL parameter<br><br>
    329      *
    330      * For a description of the parameter, refer to the general VLO
    331      * documentation.
    332      *
    333      * @param componentRegistryRESTURL the parameter
     421     * Set the value of the ComponentRegisteryRESTURL parameter<br><br>
     422     *
     423     * For a description of the parameter, refer to the general VLO
     424     * documentation.
     425     *
     426     * @param componentRegistryRESTURL the value
    334427     */
    335428    public void setComponentRegistryRESTURL(String url) {
    … …  
    338431
    339432    /**
    340      * Get the HandleServerUrl parameter<br><br>
    341      *
    342      * For a description of the parameter, refer to the general VLO
    343      * documentation.
    344      *
    345      * @return the parameter
     433     * Get the value of the HandleServerUrl parameter<br><br>
     434     *
     435     * For a description of the parameter, refer to the general VLO
     436     * documentation.
     437     *
     438     * @return the value
    346439     */
    347440    public String getHandleServerUrl() {
    … …  
    350443
    351444    /**
    352      * Set the HandleServerUrl parameter<br><br>
    353      *
    354      * For a description of the parameter, refer to the general VLO
    355      * documentation.
    356      *
    357      * @param handleServerUrl the parameter
     445     * Set the value of the HandleServerUrl parameter<br><br>
     446     *
     447     * For a description of the parameter, refer to the general VLO
     448     * documentation.
     449     *
     450     * @param handleServerUrl the value
    358451     */
    359452    public void setHandleServerUrl(String url) {
    … …  
    362455
    363456    /**
    364      * Set the IMDIBrowserUrl parameter<br><br>
    365      *
    366      * For a description of the parameter, refer to the general VLO
    367      * documentation.
    368      *
    369      * @param imdiBrowserUrl the parameter
     457     * Set the value of the IMDIBrowserUrl parameter<br><br>
     458     *
     459     * For a description of the parameter, refer to the general VLO
     460     * documentation.
     461     *
     462     * @param imdiBrowserUrl the value
    370463     */
    371464    public void setIMDIBrowserUrl(String url) {
    … …  
    374467
    375468    /**
    376      * Get ProfileSchemaUrl parameter combined with a handle<br><br>
     469     * Get the value of the ProfileSchemaUrl parameter combined with a handle<br><br>
    377470     *
    378471     * For a description of the schema, refer to the general VLO documentation.
    379472     *
    380      * @param handle the handle to be combined with the parameter
     473     * @param handle the value
    381474     */
    382475    public String getIMDIBrowserUrl(String handle) {
    … …  
    391484
    392485    /**
    393      * Get the FederatedContentSearchUrl parameter<br><br>
    394      *
    395      * For a description of the parameter, refer to the general VLO
    396      * documentation.
    397      *
    398      * @return the parameter
     486     * Get the value of the FederatedContentSearchUrl parameter<br><br>
     487     *
     488     * For a description of the parameter, refer to the general VLO
     489     * documentation.
     490     *
     491     * @return the value
    399492     */
    400493    public void setFederatedContentSearchUrl(String url) {
    … …  
    403496
    404497    /**
    405      * Set the FederatedContentSearchUrl parameter<br><br>
    406      *
    407      * For a description of the parameter, refer to the general VLO
    408      * documentation.
    409      *
    410      * @param FederatedContentSearchUrl the parameter
     498     * Set the value of the FederatedContentSearchUrl parameter<br><br>
     499     *
     500     * For a description of the parameter, refer to the general VLO
     501     * documentation.
     502     *
     503     * @param FederatedContentSearchUrl the value
    411504     */
    412505    public String getFederatedContentSearchUrl() {
    … …  
    415508
    416509    /**
    417      * Set the NationalProjectMapping parameter<br><br>
    418      *
    419      * For a description of the parameter, refer to the general VLO
    420      * documentation.
    421      *
    422      * @param nationalProjectMapping the parameter
     510     * Set the value of the NationalProjectMapping parameter<br><br>
     511     *
     512     * For a description of the parameter, refer to the general VLO
     513     * documentation.
     514     *
     515     * @param nationalProjectMapping the value
    423516     */
    424517    public void setNationalProjectMapping(String mapping) {
    … …  
    427520
    428521    /**
    429      * Get the NationalProjectMapping parameter<br><br>
    430      *
    431      * For a description of the parameter, refer to the general VLO
    432      * documentation.
    433      *
    434      * @return the parameter
     522     * Get the value of the NationalProjectMapping parameter<br><br>
     523     *
     524     * For a description of the parameter, refer to the general VLO
     525     * documentation.
     526     *
     527     * @return the value
    435528     */
    436529    public String getNationalProjectMapping() {
    … …  
    439532
    440533    /**
    441      * Get the FacetFields parameter<br><br>
    442      *
    443      * For a description of the parameter, refer to the general VLO
    444      * documentation.
    445      *
    446      * @return the parameter
     534     * Get the value of the FacetFields parameter<br><br>
     535     *
     536     * For a description of the parameter, refer to the general VLO
     537     * documentation.
     538     *
     539     * @return the value
    447540     */
    448541    public String[] getFacetFields() {
    … …  
    451544
    452545    /**
    453      * Set the FacetFields parameter<br><br>
    454      *
    455      * For a description of the parameter, refer to the general VLO
    456      * documentation.
    457      *
    458      * @param facetFields the parameter, a list of facet fields
     546     * Set the value of the FacetFields parameter<br><br>
     547     *
     548     * For a description of the parameter, refer to the general VLO
     549     * documentation.
     550     *
     551     * @param facetFields the value, a list of facet fields
    459552     */
    460553    public void setFacetFields(String[] fields) {
    … …  
    463556
    464557    /**
    465      * Get the CountryComponentUrl parameter<br><br>
    466      *
    467      * For a description of the parameter, refer to the general VLO
    468      * documentation.
    469      *
    470      * @return the parameter
     558     * Get the value of the CountryComponentUrl parameter<br><br>
     559     *
     560     * For a description of the parameter, refer to the general VLO
     561     * documentation.
     562     *
     563     * @return the value
    471564     */
    472565    public String getCountryComponentUrl() {
    … …  
    475568
    476569    /**
    477      * Set the CountryComponentUrl parameter<br><br>
    478      *
    479      * For a description of the parameter, refer to the general VLO
    480      * documentation.
    481      *
    482      * @param countryComponentUrl the parameter
     570     * Set the value of the CountryComponentUrl parameter<br><br>
     571     *
     572     * For a description of the parameter, refer to the general VLO
     573     * documentation.
     574     *
     575     * @param countryComponentUrl the value
    483576     */
    484577    public void setCountryComponentUrl(String url) {
    … …  
    487580
    488581    /**
    489      * Get the Language2LetterCodeComponentUrl parameter<br><br>
    490      *
    491      * For a description of the parameter, refer to the general VLO
    492      * documentation.
    493      *
    494      * @return the parameter
     582     * Get the value of the Language2LetterCodeComponentUrl parameter<br><br>
     583     *
     584     * For a description of the parameter, refer to the general VLO
     585     * documentation.
     586     *
     587     * @return the value
    495588     */
    496589    public String getLanguage2LetterCodeComponentUrl() {
    … …  
    499592
    500593    /**
    501      * Set the Language2LetterCodeComponentUrl parameter<br><br>
    502      *
    503      * For a description of the parameter, refer to the general VLO
    504      * documentation.
    505      *
    506      * @param language2LetterCodeComponentUrl the parameter
     594     * Set the value of the Language2LetterCodeComponentUrl parameter<br><br>
     595     *
     596     * For a description of the parameter, refer to the general VLO
     597     * documentation.
     598     *
     599     * @param language2LetterCodeComponentUrl the value
    507600     */
    508601    public void setLanguage2LetterCodeComponentUrl(String url) {
    … …  
    511604
    512605    /**
    513      * Get the Language3LetterCodeComponentUrl parameter<br><br> For a
    514      * description of the parameter, refer to the general VLO documentation.
    515      *
    516      * @return the parameter
     606     * Get the value of the Language3LetterCodeComponentUrl parameter<br><br>
     607     *
     608     * For a description of the parameter, refer to the general VLO
     609     * documentation.
     610     *
     611     * @return the value
    517612     */
    518613    public String getLanguage3LetterCodeComponentUrl() {
    … …  
    521616
    522617    /**
    523      * Set the Language3LetterCodeComponentUrl parameter<br><br> For a
    524      * description of the parameter, refer to the general VLO documentation.
    525      *
    526      * @param language3LetterCodeComponentUrl the parameter
     618     * Set the value of the Language3LetterCodeComponentUrl parameter<br><br>
     619     *
     620     * For a description of the parameter, refer to the general VLO
     621     * documentation.
     622     *
     623     * @param language3LetterCodeComponentUrl the value
    527624     */
    528625    public void setLanguage3LetterCodeComponentUrl(String url) {
    … …  
    531628
    532629    /**
    533      * Get the SilToISO639CodesUrl parameter<br><br>
    534      *
    535      * For a description of the parameter, refer to the general VLO
    536      * documentation.
    537      *
    538      * @return the parameter
     630     * Get the value of the SilToISO639CodesUrl parameter<br><br>
     631     *
     632     * For a description of the parameter, refer to the general VLO
     633     * documentation.
     634     *
     635     * @return the value
    539636     */
    540637    public String getSilToISO639CodesUrl() {
    … …  
    543640
    544641    /**
    545      * Set the SilToISO639CodesUrl parameter<br><br>
    546      *
    547      * For a description of the parameter, refer to the general VLO
    548      * documentation.
    549      *
    550      * @param silToISO639CodesUrl the parameter
     642     * Set the value of the SilToISO639CodesUrl parameter<br><br>
     643     *
     644     * For a description of the parameter, refer to the general VLO
     645     * documentation.
     646     *
     647     * @param silToISO639CodesUrl the value
    551648     */
    552649    public void setSilToISO639CodesUrl(String url) {

  • vlo/branches/vlo-2.13-param/vlo_webapp/src/main/java/eu/clarin/cmdi/vlo/importer/MetadataImporter.java

    r2496 r2604  
    11package eu.clarin.cmdi.vlo.importer;
    22
     3import eu.clarin.cmdi.vlo.CommonUtils;
     4import eu.clarin.cmdi.vlo.Configuration;
     5import eu.clarin.cmdi.vlo.FacetConstants;
     6import eu.clarin.cmdi.vlo.config.DataRoot;
     7import eu.clarin.cmdi.vlo.config.VloConfig;
    38import java.io.File;
    49import java.io.IOException;
    … …  
    1116import java.util.Map;
    1217import java.util.Set;
    13 
    1418import org.apache.commons.io.FileUtils;
    1519import org.apache.solr.client.solrj.SolrServerException;
    … …  
    2125import org.slf4j.Logger;
    2226import org.slf4j.LoggerFactory;
    23 import org.springframework.beans.factory.BeanFactory;
    24 import org.springframework.context.support.ClassPathXmlApplicationContext;
    25 
    26 import eu.clarin.cmdi.vlo.CommonUtils;
    27 import eu.clarin.cmdi.vlo.Configuration;
    28 import eu.clarin.cmdi.vlo.FacetConstants;
    2927
    3028
    … …  
    3230 * The main metadataImporter class. Also contains the main function.
    3331 *
    34  * The metadataimporter reads all the config files and then, for each metadatafile in each defined directory structure parses and imports them as defined in the configuration.
    35  * The startImport function starts the importing and so on.
     32 * The metadataimporter reads all the config files and then, for each
     33 * metadatafile in each defined directory structure parses and imports them as
     34 * defined in the configuration. The startImport function starts the importing
     35 * and so on.
    3636 */
    3737
    … …  
    5757     */
    5858    private StreamingUpdateSolrServer solrServer;
    59 
    60     /**
    61      * Defines the post-processor associations.
    62      * At import, for each facet value, this map is checked and all postprocessors associated with the facet _type_ are applied to the value before storing the new value in the solr document.
     59    /**
     60     * Defines the post-processor associations. At import, for each facet value,
     61     * this map is checked and all postprocessors associated with the facet
     62     * _type_ are applied to the value before storing the new value in the solr
     63     * document.
    6364     */
    6465    final static Map<String, PostProcessor> POST_PROCESSORS = new HashMap<String, PostProcessor>();
    … …  
    7172        POST_PROCESSORS.put(FacetConstants.FIELD_CLARIN_PROFILE, new CMDIComponentProfileNamePostProcessor());
    7273    }
     74   
     75    // VLO configuration
     76    VloConfig config;
     77   
     78    /**
     79     * Constructor
     80     *
     81     * @param
     82     */
     83    public MetadataImporter (VloConfig config){
     84        this.config = config;
     85    }
    7386
    7487    /**
    … …  
    7891    private final Set<String> processedIds = new HashSet<String>();
    7992    /**
    80      * Some caching for solr documents (we are more efficient if we ram a whole bunch to the solr server at once.
     93     * Some caching for solr documents (we are more efficient if we ram a whole
     94     * bunch to the solr server at once.
    8195     */
    8296    protected List<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
    83     /**
    84      * Config.
    85      */
    86     private final ImporterConfig config;
    8797
    8898    // SOME STATS
    … …  
    94104
    95105    /**
    96      * Constructor, wants to know the config.
    97      * @param config the config.
    98      */
    99     public MetadataImporter(ImporterConfig config) {
    100         this.config = config;
    101     }
    102 
    103     /**
    104      * Retrieve all files with VALID_CMDI_EXTENSIONS from all DataRoot entries and starts processing for every single file
     106     * Retrieve all files with VALID_CMDI_EXTENSIONS from all DataRoot entries
     107     * and starts processing for every single file
     108     *
    105109     * @throws MalformedURLException
    106110     */
    … …  
    112116        try {
    113117            // Delete the whole Solr db
    114             if (config.isDeleteAllFirst()) {
     118            if (VloConfig.get().isDeleteAllFirst()) {
    115119                LOG.info("Deleting original data...");
    116120                solrServer.deleteByQuery("*:*");
    … …  
    162166
    163167    /**
    164      * Check a List of DataRoots for existence of RootFile (typically parent directory of metadata files)
     168     * Check a List of DataRoots for existence of RootFile (typically parent
     169     * directory of metadata files)
     170     *
    165171     * @return
    166172     */
    167173    private List<DataRoot> checkDataRoots() {
    168         List<DataRoot> dataRoots = config.getDataRoots();
     174        List<DataRoot> dataRoots = VloConfig.get().getDataRoots();
    169175        for (DataRoot dataRoot : dataRoots) {
    170176            if (!dataRoot.getRootFile().exists()) {
    … …  
    177183
    178184    /**
    179      * Get the rootFile or all files with VALID_CMDI_EXTENSIONS if rootFile is a directory
     185     * Get the rootFile or all files with VALID_CMDI_EXTENSIONS if rootFile is a
     186     * directory
     187     *
    180188     * @param rootFile
    181      * @return List with the rootFile or all contained files if rootFile is a directory
     189     * @return List with the rootFile or all contained files if rootFile is a
     190     * directory
    182191     */
    183192    private List<File> getFilesFromDataRoot(File rootFile) {
    … …  
    194203    /**
    195204     * Initialize SolrServer as specified in configuration file
     205     *
    196206     * @throws MalformedURLException
    197207     */
    … …  
    210220    /**
    211221     * Process single CMDI file with CMDIDataProcessor
     222     *
    212223     * @param file CMDI input file
    213224     * @param dataOrigin
    … …  
    248259    /**
    249260     * Check id for validness
     261     *
    250262     * @param id
    251263     * @return true if id is acceptable, false otherwise
    … …  
    256268
    257269    /**
    258      * Adds some additional information from DataRoot to solrDocument, add solrDocument to document list, submits list to SolrServer every 1000 files
     270     * Adds some additional information from DataRoot to solrDocument, add
     271     * solrDocument to document list, submits list to SolrServer every 1000
     272     * files
     273     *
    259274     * @param solrDocument
    260275     * @param cmdiData
    … …  
    293308
    294309    /**
    295      * Adds two fields FIELD_RESOURCE_TYPE and FIELD_RESOURCE. The Type can be specified in the "ResourceType" element of an imdi file or
    296      * possibly overwritten by some more specific xpath (as in the LRT cmdi files). So if a type is overwritten and already in the
    297      * solrDocument we take that type.
     310     * Adds two fields FIELD_RESOURCE_TYPE and FIELD_RESOURCE. The Type can be
     311     * specified in the "ResourceType" element of an imdi file or possibly
     312     * overwritten by some more specific xpath (as in the LRT cmdi files). So if
     313     * a type is overwritten and already in the solrDocument we take that type.
    298314     */
    299315    private void addResourceData(SolrInputDocument solrDocument, CMDIData cmdiData) {
    … …  
    324340
    325341    /**
    326      * Send current list of SolrImputDocuments to SolrServer and clears list afterwards
     342     * Send current list of SolrImputDocuments to SolrServer and clears list
     343     * afterwards
     344     *
    327345     * @throws SolrServerException
    328346     * @throws IOException
    … …  
    340358    /**
    341359     * Builds suggester index for autocompletion
     360     *
    342361     * @throws SolrServerException
    343362     * @throws MalformedURLException
    … …  
    357376     */
    358377    public static void main(String[] args) throws IOException {
    359         BeanFactory factory = new ClassPathXmlApplicationContext(new String[] { Configuration.CONFIG_FILE });
    360         factory.getBean("configuration");
    361         Configuration cfg = Configuration.getInstance();
    362 
    363         BeanFactory factory2 = new ClassPathXmlApplicationContext(new String[] { cfg.getImporterConfigFile() } );
    364 
    365         ImporterConfig config = (ImporterConfig) factory2.getBean("importerConfig", ImporterConfig.class);
    366 
    367         MetadataImporter importer = new MetadataImporter(config);
    368         importer.startImport();
    369         if (config.isPrintMapping()) {
    370             File file = new File("xsdMapping.txt");
    371             FacetMappingFactory.printMapping(file);
    372             LOG.info("Printed facetMapping in " + file);
    373         }
    374     }
    375 
     378
     379        String fileName;
     380
     381        VloConfig config;
     382       
     383        if (args.length > 0) {
     384
     385            // get the file name from the command line
     386            fileName = VloConfig.class.getResource(args[0]).getFile();
     387           
     388            // test for file existence
     389
     390            // optionally, modify the configuration before starting the importer
     391
     392            config = VloConfig.readConfig(fileName);
     393           
     394            MetadataImporter importer = new MetadataImporter(config);
     395           
     396            importer.startImport();
     397           
     398            if (config.isPrintMapping()) {
     399                File file = new File("xsdMapping.txt");
     400                FacetMappingFactory.printMapping(file);
     401                LOG.info("Printed facetMapping in " + file);
     402            }
     403        }
     404    }
    376405}

  • vlo/branches/vlo-2.13-param/vlo_webapp/src/main/resources/VloConfig.xml

    r2598 r2604  
    11<webAppConfig>
    2    <!-- <solrUrl>${eu.clarin.cmdi.vlo.solrUrl}</solrUrl> -->
    3    <solrUrl>http://localhost:8084/vlo_solr/</solrUrl>
    4    <vloHomeLink>http://www.clarin.eu/vlo</vloHomeLink>
    5    <profileSchemaUrl>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/profiles/</profileSchemaUrl>
    6    <componentRegistryRESTURL>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/profiles/</componentRegistryRESTURL>
    7    <handleServerUrl>http://hdl.handle.net/</handleServerUrl>
    8    <imdiBrowserUrl>http://corpus1.mpi.nl/ds/imdi_browser?openpath=</imdiBrowserUrl>
    9    <nationalProjectMapping>nationalProjectsMapping.xml</nationalProjectMapping>
    10    <facetFields length="6">
    11       <facetField>collection</facetField>
    12       <facetField>continent</facetField>
    13       <facetField>organisation</facetField>
    14       <facetField>genre</facetField>
    15       <facetField>country</facetField>
    16       <facetField>language</facetField>
    17    </facetFields>
    18    <countryComponentUrl>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/components/clarin.eu:cr1:c_1271859438104/xml</countryComponentUrl>
    19    <language2LetterCodeComponentUrl>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/components/clarin.eu:cr1:c_1271859438109/xml</language2LetterCodeComponentUrl>
    20    <language3LetterCodeComponentUrl>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/components/clarin.eu:cr1:c_1271859438110/xml</language3LetterCodeComponentUrl>
    21    <silToISO639CodesUrl>http://www.clarin.eu/cmd/xslt/sil_to_iso6393.xml</silToISO639CodesUrl>
    22    <FederatedContentSearchUrl>http://weblicht.sfs.uni-tuebingen.de/Aggregator/</FederatedContentSearchUrl>
     2   
     3    <deleteAllFirst>false</deleteAllFirst>
     4   
     5    <printMapping>false</printMapping>
     6   
     7    <dataRoots>
     8        <DataRoot>
     9            <originName>MPI IMDI Archive
     10            </originName>
     11            <rootFile>/lat/apache/htdocs/oai-harvester/mpi-self-harvest/harvested/results/cmdi/</rootFile>           
     12            <prefix>http://catalog.clarin.eu/</prefix>
     13            <tostrip>/lat/apache/htdocs/</tostrip>
     14            <deleteFirst>false</deleteFirst>
     15        </DataRoot>
     16
     17        <DataRoot>
     18            <originName>CMDI Providers</originName>
     19            <rootFile>/lat/apache/htdocs/oai-harvester/cmdi-providers/harvested/results/cmdi/</rootFile>           
     20            <prefix>http://catalog.clarin.eu/</prefix>
     21            <tostrip>/lat/apache/htdocs/</tostrip>
     22            <deleteFirst>false</deleteFirst>
     23        </DataRoot>
     24
     25        <DataRoot>
     26            <originName>OLAC Metadata Providers</originName>
     27            <rootFile>/lat/apache/htdocs/oai-harvester/olac-and-dc-providers/harvested/results/cmdi/</rootFile>           
     28            <prefix>http://catalog.clarin.eu/</prefix>
     29            <tostrip>/lat/apache/htdocs/</tostrip>
     30            <deleteFirst>false</deleteFirst>
     31        </DataRoot>
     32    </dataRoots>
     33   
     34    <solrUrl>http://localhost:8084/vlo_solr/</solrUrl>
     35   
     36    <vloHomeLink>http://www.clarin.eu/vlo</vloHomeLink>
     37   
     38    <profileSchemaUrl>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/profiles/</profileSchemaUrl>
     39   
     40    <componentRegistryRESTURL>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/profiles/</componentRegistryRESTURL>
     41   
     42    <handleServerUrl>http://hdl.handle.net/</handleServerUrl>
     43   
     44    <imdiBrowserUrl>http://corpus1.mpi.nl/ds/imdi_browser?openpath=</imdiBrowserUrl>
     45   
     46    <nationalProjectMapping>nationalProjectsMapping.xml</nationalProjectMapping>
     47   
     48    <facetFields length="6">
     49        <facetField>collection</facetField>
     50        <facetField>continent</facetField>
     51        <facetField>organisation</facetField>
     52        <facetField>genre</facetField>
     53        <facetField>country</facetField>
     54        <facetField>language</facetField>
     55    </facetFields>
     56   
     57    <countryComponentUrl>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/components/clarin.eu:cr1:c_1271859438104/xml</countryComponentUrl>
     58
     59    <language2LetterCodeComponentUrl>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/components/clarin.eu:cr1:c_1271859438109/xml</language2LetterCodeComponentUrl>
     60   
     61    <language3LetterCodeComponentUrl>http://catalog.clarin.eu/ds/ComponentRegistry/rest/registry/components/clarin.eu:cr1:c_1271859438110/xml</language3LetterCodeComponentUrl>
     62   
     63    <silToISO639CodesUrl>http://www.clarin.eu/cmd/xslt/sil_to_iso6393.xml</silToISO639CodesUrl>
     64   
     65    <FederatedContentSearchUrl>http://weblicht.sfs.uni-tuebingen.de/Aggregator/</FederatedContentSearchUrl>
    2366</webAppConfig>

  • vlo/branches/vlo-2.13-param/vlo_webapp/src/test/java/eu/clarin/cmdi/vlo/importer/MetadataImporterTest.java

    r1437 r2604  
    11package eu.clarin.cmdi.vlo.importer;
    22
    3 import static org.junit.Assert.assertEquals;
    4 
     3import eu.clarin.cmdi.vlo.FacetConstants;
     4import eu.clarin.cmdi.vlo.config.DataRoot;
     5import eu.clarin.cmdi.vlo.config.VloConfig;
    56import java.io.File;
    67import java.io.IOException;
    … …  
    1011import java.util.Collections;
    1112import java.util.List;
    12 
    1313import org.apache.solr.client.solrj.SolrServerException;
    1414import org.apache.solr.common.SolrInputDocument;
     15import static org.junit.Assert.assertEquals;
    1516import org.junit.Test;
    16 
    17 import eu.clarin.cmdi.vlo.FacetConstants;
    1817
    1918public class MetadataImporterTest extends ImporterTestcase {
    … …  
    236235    private List<SolrInputDocument> importData(File rootFile) throws MalformedURLException {
    237236        final List<SolrInputDocument> result = new ArrayList<SolrInputDocument>();
    238         ImporterConfig config = createConfig(rootFile);
    239         MetadataImporter importer = new MetadataImporter(config) {
     237       
     238        String fileName = VloConfig.class.getResource("/VloConfig.xml").getFile();
     239       
     240        VloConfig testConfig;
     241       
     242        /**
     243         * Problem: at the moment the readTestConfig method is not prepared for
     244         * a message from the importer. May rename readTestConfig to
     245         * readWebAppTestConfig, so that we have space for a
     246         * readImporterTestConfig also.
     247         *
     248         * The best thing would be to have a directory for testing inside the
     249         * package. You can always use an external configuration file for more
     250         * elaborate testing.test directory inside the package.
     251         */
     252
     253        testConfig = VloConfig.readTestConfig(fileName);
     254       
     255        // modify the test configuration a bit
     256       
     257        testConfig = modifyConfig(testConfig, rootFile);
     258       
     259        // ... and also the importer itself
     260       
     261        MetadataImporter importer;
     262        importer = new MetadataImporter(testConfig) {
    240263            @Override
    241264            protected void initSolrServer() throws MalformedURLException {
    … …  
    245268            @Override
    246269            protected void sendDocs() throws SolrServerException, IOException {
    247                 //overriding here so we can test the docs
     270            //overriding here so we can test the docs
    248271                result.addAll(this.docs);
    249272                docs = new ArrayList<SolrInputDocument>();
    … …  
    254277    }
    255278
    256     private ImporterConfig createConfig(File rootFile) {
    257         ImporterConfig config = new ImporterConfig();
     279    private VloConfig modifyConfig(VloConfig config, File rootFile) {
    258280        DataRoot dataRoot = new DataRoot();
    259281        dataRoot.setDeleteFirst(false);//cannot delete not using real solrServer

  • vlo/branches/vlo-2.13-param/vlo_webapp/src/test/java/eu/clarin/cmdi/vlo/pages/FacetBoxPanelTest.java

    r2600 r2604  
    1818    public void setUp() {
    1919        WicketTester wicketTester;
     20       
    2021        String fileName = VloConfig.class.getResource("/VloConfig.xml").getFile();
    2122       
Note: See TracChangeset for help on using the changeset viewer.