Changeset 2527

Timestamp:

01/31/13 15:55:57 (11 years ago)

Author:

yana

Message:

Logging organized, progress info about waiting for search results added, SRUThreadedClient is one per application now.

Location:

SRUAggregator/trunk

Files:

• pom.xml (modified) (2 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/app/Aggregator.java (modified) (11 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/app/DesktopDestroyedListener.java (modified) (2 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/app/WebAppListener.java (moved) (moved from SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/WebAppDestroyedListener.java) (2 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/data/CenterRegistry.java (modified) (6 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/data/Corpus.java (modified) (3 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/data/Endpoint.java (modified) (4 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/data/Institution.java (modified) (4 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/data/SearchResult.java (modified) (3 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/sresult/SearchResultRecordRenderer.java (modified) (8 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/sresult/SearchResultsController.java (modified) (18 diffs)
• src/main/webapp/WEB-INF/zk.xml (modified) (1 diff)
• src/main/webapp/index.zul (modified) (3 diffs)

SRUAggregator/trunk/pom.xml

@@ -4 +4 @@
     <groupId>eu.clarin.sru.fcs</groupId>
     <artifactId>aggregator</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <version>1.0</version>
     <properties>
         <endorsed.dir>${project.build.directory}/endorsed</endorsed.dir>
@@ -54 +54 @@
             <artifactId>sru-client</artifactId>
             <version>0.9.0</version>
+        </dependency>
+        <dependency>
+            <groupId>eu.clarin.weblicht</groupId>
+            <artifactId>wlfxb</artifactId>
+            <version>1.2.2</version>
         </dependency>
     </dependencies>

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/Aggregator.java

@@ -3 +3 @@
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
-import eu.clarin.sru.client.SRUClientException;
-import eu.clarin.sru.client.SRUExplainRequest;
-import eu.clarin.sru.client.SRUExplainResponse;
-import eu.clarin.sru.client.SRURecord;
-import eu.clarin.sru.client.SRUSearchRetrieveRequest;
-import eu.clarin.sru.client.SRUSearchRetrieveResponse;
-import eu.clarin.sru.client.SRUSurrogateRecordData;
-import eu.clarin.sru.client.SRUThreadedClient;
 import eu.clarin.sru.client.SRUVersion;
-import eu.clarin.sru.client.fcs.ClarinFCSRecordData;
-import eu.clarin.sru.client.fcs.ClarinFCSRecordParser;
-import eu.clarin.sru.client.fcs.DataView;
-import eu.clarin.sru.client.fcs.DataViewKWIC;
-import eu.clarin.sru.client.fcs.Resource;
 import eu.clarin.sru.fcs.aggregator.data.CenterRegistry;
 import eu.clarin.sru.fcs.aggregator.sparam.CorpusTreeModel;
@@ -23 +10 @@
 import java.lang.reflect.Type;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.logging.*;
-import org.zkoss.zhtml.Filedownload;
 import org.zkoss.zk.ui.Component;
-import org.zkoss.zk.ui.Desktop;
 import org.zkoss.zk.ui.Executions;
 import org.zkoss.zk.ui.event.Event;
-import org.zkoss.zk.ui.event.EventListener;
 import org.zkoss.zk.ui.select.SelectorComposer;
 import org.zkoss.zk.ui.select.annotation.Listen;
 import org.zkoss.zk.ui.select.annotation.Wire;
-import org.zkoss.zul.Auxhead;
-import org.zkoss.zul.Auxheader;
 import org.zkoss.zul.Button;
-import org.zkoss.zul.Checkbox;
-import org.zkoss.zul.Column;
-import org.zkoss.zul.Columns;
 import org.zkoss.zul.Combobox;
 import org.zkoss.zul.Comboitem;
-import org.zkoss.zul.Grid;
 import org.zkoss.zul.Groupbox;
 import org.zkoss.zul.Iframe;
 import org.zkoss.zul.Label;
+import org.zkoss.zul.Menuitem;
 import org.zkoss.zul.Messagebox;
-import org.zkoss.zul.Row;
-import org.zkoss.zul.Rows;
-import org.zkoss.zul.Separator;
 import org.zkoss.zul.Textbox;
 import org.zkoss.zul.Tree;
 import org.zkoss.zul.Treechildren;
 import org.zkoss.zul.Treeitem;
-import org.zkoss.zul.Vbox;
 import org.zkoss.zul.Window;
 import org.zkoss.zul.event.ZulEvents;
@@ -72 +43 @@
 public class Aggregator extends SelectorComposer<Component> {
 
-    private static final Logger logger = Logger.getLogger("FCS-AGGREGATOR");
+    private static final Logger logger = Logger.getLogger(Aggregator.class.getName());
+    
 //    @Wire
 //    private Grid anzeigeGrid;
@@ -79 +51 @@
     @Wire
     private Combobox languageSelect;
-    @Wire
-    private Button searchButton;
-    @Wire
-    private Groupbox allCorpora;
+//    @Wire
+//    private Button searchButton;
+//    @Wire
+//    private Groupbox allCorpora;
 //    @Wire
 //    private Comboitem german;
@@ -89 +61 @@
     @Wire
     private Window resultsBox;
-    @Wire
-    private Button selectAll;
-    @Wire
-    private Button deselectAll;
-    @Wire
-    private Window mainWindow;
+//    @Wire
+//    private Button selectAll;
+//    @Wire
+//    private Button deselectAll;
+//    @Wire
+//    private Window mainWindow;
     @Wire
     private Combobox maximumRecordsSelect;
-    @Wire
-    private Button addForeignEndpoint;
+//    @Wire
+//    private Button addForeignEndpoint;
     @Wire
     Combobox foreignEndpointSelect;
     @Wire
     private Tree tree;
+    @Wire
+    private Label searchResultsProgress;
+    
     private Map<String, List<String>> xAggregationContext;
+    private SRUVersion version = SRUVersion.VERSION_1_2;
     private SearchResultsController searchResultsController;
     private CenterRegistry registry;
@@ -109 +85 @@
     @Override
     public void doAfterCompose(Component comp) throws Exception {
+        
         super.doAfterCompose(comp);
 
@@ -114 +91 @@
 
         languageSelect.setSelectedItem(anyLanguage);
-        searchResultsController = new SearchResultsController(resultsBox);
-        // assign it to desktop, so that it can be accessed to be shutdown when the desktop is destroyed
-        // TODO the registry/tree also has to be shutdown properly???
+        
+        searchResultsController = new SearchResultsController(resultsBox, searchResultsProgress);
+        // assign the search controller to desktop, so that it can be accessed to be shutdown when the desktop is destroyed
         Executions.getCurrent().getDesktop().setAttribute(searchResultsController.getClass().getSimpleName(), searchResultsController);
+        // also add it to the list of actice controllers of the web application, so that they can be shutdown when the application stops
+        Set<SearchResultsController> activeControllers = (Set<SearchResultsController>) Executions.getCurrent().getDesktop().getWebApp().getAttribute(WebAppListener.ACTIVE_SEARCH_CONTROLLERS);
+        activeControllers.add(searchResultsController);
         
         registry = new CenterRegistry();
@@ -158 +138 @@
     public void onExecuteSearch(Event ev) {
         int maxRecords = Integer.parseInt(maximumRecordsSelect.getValue());
-        logger.info("Executing Search.");
-        searchResultsController.executeSearch(tree.getSelectedItems(), maxRecords, searchString.getText());
+        searchResultsController.executeSearch(tree.getSelectedItems(), maxRecords, searchString.getText(), version);
     }
  
@@ -185 +164 @@
     @Listen("onClick=#exportResultsCSV")
     public void onExportResultsCSV(Event ev) {
-
-        int i, i2, i3;
-        String temp = "";
-        boolean somethingToExport = false;
-
-        for (i = 0; i < resultsBox.getChildren().size(); i++) {
-            if (resultsBox.getChildren().get(i) instanceof Grid) {
-                somethingToExport = true;
-                Grid aGrid = (Grid) resultsBox.getChildren().get(i);
-                Rows rows = aGrid.getRows();
-
-                for (i2 = 0; i2 < rows.getChildren().size(); i2++) {
-                    Row r = (Row) rows.getChildren().get(i2);
-
-                    for (i3 = 0; i3 < r.getChildren().size(); i3++) {
-                        Label l = (Label) r.getChildren().get(i3);
-                        temp = temp + "\"" + l.getValue().replace("\"", "QUOTE") + "\"";
-                        if (i3 < r.getChildren().size() - 1) {
-                            temp = temp + ",";
-                        } //if i3
-                    } //for i3
-                    temp = temp + "\n";
-                } // for i2
-            } // if grid
-
-        } // for i ...
-
-        if (somethingToExport) {
-
-            Filedownload.save(temp, "text/plain", "ClarinDFederatedContentSearch.csv");
-        } else {
-            Messagebox.show("Nothing to export!");
-        }
+        searchResultsController.exportCSV();
     }
 
     @Listen("onClick=#exportResultsTCF")
     public void onExportResultsTCF(Event ev) {
-
-        int i, i2, i3;
-        String temp = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><D-Spin xmlns=\"http://www.dspin.de/data\" version=\"0.4\">\n<MetaData xmlns=\"http://www.dspin.de/data/metadata\">\n";
-        temp = temp + "<source>CLARIN-D Federated Content Search</source>\n</MetaData>\n  <TextCorpus xmlns=\"http://www.dspin.de/data/textcorpus\" lang=\"de\">\n<text>";
-
-
-        boolean somethingToExport = false;
-
-        for (i = 0; i < resultsBox.getChildren().size(); i++) {
-            if (resultsBox.getChildren().get(i) instanceof Grid) {
-                somethingToExport = true;
-                Grid aGrid = (Grid) resultsBox.getChildren().get(i);
-                Rows rows = aGrid.getRows();
-
-                for (i2 = 0; i2 < rows.getChildren().size(); i2++) {
-                    Row r = (Row) rows.getChildren().get(i2);
-
-                    for (i3 = 0; i3 < r.getChildren().size(); i3++) {
-                        Label l = (Label) r.getChildren().get(i3);
-                        temp = temp + l.getValue() + " ";
-                    } //for i3
-                    temp = temp + "\n";
-                } // for i2
-            } // if grid
-
-        } // for i ...
-
-        if (somethingToExport) {
-            temp = temp + "</text>\n</TextCorpus>\n</D-Spin>";
-            Filedownload.save(temp, "text/tcf+xml", "ClarinDFederatedContentSearch.xml");
-        } else {
-            Messagebox.show("Nothing to export!");
-        }
+        searchResultsController.exportTCF();
     }
     
@@ -263 +178 @@
 
     private void processParameters() {
+        
         String[] paramValue;
         String contextJson = null;
 
-        //TODO use them???
-        String operationString = null;
-        String versionString = null;
-
         paramValue = Executions.getCurrent().getParameterMap().get("query");
         if (paramValue != null) {
-            searchString.setValue(paramValue[0]);
+            searchString.setValue(paramValue[0].trim());
+            logger.log(Level.INFO, "Received parameter: query[{0}]",  searchString.getValue());
         }
         paramValue = Executions.getCurrent().getParameterMap().get("operation");
         if (paramValue != null) {
-            operationString = paramValue[0];
+            String operationString = paramValue[0].trim();
+            logger.log(Level.INFO, "Received parameter: operation[{0}]", operationString);
             if (!operationString.equals("searchRetrieve")) {
-                logger.severe("Not supported operation: " + operationString);
+                logger.log(Level.SEVERE, "Not supported operation: {0}", operationString);
+                Messagebox.show("CLARIN-D Federated Content Search Aggregator\n\nVersion 0.0.1", "FCS", 0, Messagebox.INFORMATION);
             }
         }
         paramValue = Executions.getCurrent().getParameterMap().get("version");
         if (paramValue != null) {
-            versionString = paramValue[0];
+            String versionString = paramValue[0].trim();
+            logger.log(Level.INFO, "Received parameter: version[{0}]", versionString);
+            if (versionString.equals("1.2")) {
+                version = SRUVersion.VERSION_1_2;
+            } else if (versionString.equals("1.1")) {
+                version = SRUVersion.VERSION_1_1;
+            } else {
+                logger.log(Level.SEVERE, "Not supported SRU version: {0}", versionString);
+                Messagebox.show("SRU Version " + version + " not supported", "FCS", 0, Messagebox.INFORMATION);
+            }
         }
         paramValue = Executions.getCurrent().getParameterMap().get("x-aggregation-context");
         if (paramValue != null) {
-            contextJson = paramValue[0];
-        }
-
-        logger.info("query: " + searchString.getValue());
-        logger.info("operation: " + operationString);
-        logger.info("version: " + versionString);
-        logger.info("x-aggregation-context: " + contextJson);
+            contextJson = paramValue[0].trim();
+            logger.log(Level.INFO, "Received parameter: x-aggregation-context[{0}]", contextJson);
+        }
 
         if (contextJson != null) {
@@ -299 +219 @@
             Type mapType = new TypeToken<LinkedHashMap<String, ArrayList<String>>>() {
             }.getType();
+            try {
             this.xAggregationContext = gson.fromJson(contextJson, mapType);
-            //System.out.println("selectedEndpoints: " + selectedEndpoints);
-            //selectEndpoints(selectedEndpoints);
-        }
-
+            } catch (Exception ex) {
+                logger.log(Level.SEVERE, "Error parsing JSON from x-aggregation-context:\n {0}\n {1}", new String[]{ex.getMessage(), contextJson});
+                Messagebox.show("Error in x-aggregation-context parameter", "FCS", 0, Messagebox.INFORMATION);
+            }
+        }
 
     }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/DesktopDestroyedListener.java

@@ -3 +3 @@
 
 import eu.clarin.sru.fcs.aggregator.sresult.SearchResultsController;
+import java.util.Set;
 import java.util.logging.Logger;
 import org.zkoss.zk.ui.Desktop;
@@ -12 +13 @@
  */
 public class DesktopDestroyedListener implements DesktopCleanup {
+    
+    private static final Logger logger = Logger.getLogger(DesktopCleanup.class.getName());
 
     @Override
     public void cleanup(Desktop desktop) {
-
+        logger.fine("Cleaning up desktop...");
         Object recordsController = desktop.getAttribute(SearchResultsController.class.getSimpleName());
+        Set<SearchResultsController> activeControllers = (Set<SearchResultsController>) desktop.getWebApp().getAttribute(WebAppListener.ACTIVE_SEARCH_CONTROLLERS);
         if (recordsController != null) {
-            Logger.getLogger(DesktopDestroyedListener.class.getName()).info("Cleaning up desktop");
-            ((SearchResultsController) recordsController).shutdown();
+            SearchResultsController srController = (SearchResultsController) recordsController;
+            srController.shutdown();
+            activeControllers.remove(srController);
         }
     }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/WebAppListener.java

@@ -1 +1 @@
 package eu.clarin.sru.fcs.aggregator.app;
 
-
+import eu.clarin.sru.client.SRUClientException;
+import eu.clarin.sru.client.SRUThreadedClient;
+import eu.clarin.sru.client.fcs.ClarinFCSRecordParser;
+import eu.clarin.sru.fcs.aggregator.sresult.SearchResultsController;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.zkoss.zk.ui.WebApp;
 import org.zkoss.zk.ui.util.WebAppCleanup;
+import org.zkoss.zk.ui.util.WebAppInit;
 
 /**
@@ -10 +17 @@
  * @author Yana Panchenko
  */
-public class WebAppDestroyedListener implements WebAppCleanup {
+public class WebAppListener implements WebAppInit, WebAppCleanup {
+
+    public static String ACTIVE_SEARCH_CONTROLLERS = "ACTIVE_SEARCH_CONTROLLERS";
+    public static String SHARED_SRU_CLIENT = "SHARED_SRU_CLIENT";
+    private static final Logger logger = Logger.getLogger(WebAppListener.class.getName());
 
     @Override
-    public void cleanup(WebApp wapp) throws Exception {
-        //TODO
-        Logger.getLogger(WebAppDestroyedListener.class.getName()).info("TODO: WebAppDestroyedListener");
+    public void init(WebApp webapp) throws Exception {
+
+        //int endOfClassesSharedPartOfName = WebAppListener.class.getPackage().getName().lastIndexOf(".");
+        //Logger.getLogger(WebAppListener.class.getPackage().getName().substring(0, endOfClassesSharedPartOfName)).setLevel(Level.FINE);
+        
+        logger.info("Aggregator is starting...");
+        
+        Set<SearchResultsController> activeControllers = new HashSet<SearchResultsController>();
+        webapp.setAttribute(ACTIVE_SEARCH_CONTROLLERS, activeControllers);
+
+        SRUThreadedClient searchClient = new SRUThreadedClient();
+        try {
+            searchClient.registerRecordParser(new ClarinFCSRecordParser());
+            webapp.setAttribute(WebAppListener.SHARED_SRU_CLIENT, searchClient);
+        } catch (SRUClientException e) {
+            logger.log(Level.SEVERE, "SRU Client Parser registration failed", e);
+        }
+    }
+
+    @Override
+    public void cleanup(WebApp webapp) throws Exception {
+        logger.info("Aggregator is shutting down...");
+
+        Set<SearchResultsController> activeControllers = (Set<SearchResultsController>) webapp.getAttribute(ACTIVE_SEARCH_CONTROLLERS);
+        for (SearchResultsController activeController : activeControllers) {
+            activeController.shutdown();
+        }
+        SRUThreadedClient searchClient = (SRUThreadedClient) webapp.getAttribute(WebAppListener.SHARED_SRU_CLIENT);
+
+        // with shutdown() there are memory leaks when web app stops even if all requests have been processed;
+        // with shutdownNow() there are memory leaks when web app stops only if not all requests have been processed
+        searchClient.shutdownNow();
     }
 }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/data/CenterRegistry.java

@@ -7 +7 @@
 
 import eu.clarin.sru.fcs.aggregator.sparam.CorpusTreeNode;
+import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
@@ -29 +30 @@
 public class CenterRegistry implements CorpusTreeNode {
     
-    private static final Logger logger = Logger.getLogger("FCS-AGGREGATOR");
+    private static final Logger logger = Logger.getLogger(CenterRegistry.class.getName());
 
     private static final String crStartpoint = "http://130.183.206.32/restxml/";
 
     //https://centerregistry-clarin.esc.rzg.mpg.de/restxml/
-    //http://130.183.206.32/restxml/
     
     private boolean hasChildrenLoaded = false;
@@ -51 +51 @@
         }
         hasChildrenLoaded = true;
-        InputStream is;
+        InputStream is = null;
         URL u;
         NodeList instituteNames;
@@ -57 +57 @@
         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
         DocumentBuilder builder;
+        URLConnection urlConn;
         try {
             u = new URL(crStartpoint);
-            URLConnection urlConn = u.openConnection();
+            urlConn = u.openConnection();
 
             //HttpsURLConnection urlConn = (HttpsURLConnection) u.openConnection();
@@ -75 +76 @@
             org.w3c.dom.Document document = builder.parse(is);
 
-            is.close();
+            
             instituteNames = evaluateXPath("//Centername", document);
             institutionsUrls = evaluateXPath("//Center_id_link", document);
@@ -90 +91 @@
 
         } catch (Exception ex) {
-            logger.log(Level.SEVERE, "Error accessing central repository information", ex);
+            logger.log(Level.SEVERE, "Error accessing central registry information {0} {1}", new String[]{ex.getClass().getName(), ex.getMessage()});
+        } finally {
+            if (is != null) {
+                try {
+                    is.close();
+                } catch (IOException ex) {
+                    logger.log(Level.SEVERE, ex.getMessage());
+                }
+            }
         }
-        logger.info(String.format("Number of Centers: %s", centers.size()));
+        logger.log(Level.FINE, "Number of Centers: {0}", centers.size());
 
     }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/data/Corpus.java

@@ -27 +27 @@
     private boolean hasChildrenLoaded = false;
     private Endpoint endpoint;
-    // private String endpointUrl;
+    
+    private static final Logger logger = Logger.getLogger(Corpus.class.getName());
     
     public Corpus(Endpoint endpoint) {
@@ -135 +136 @@
             corporaResponse = sruClient.scan(corporaRequest);
         } catch (SRUClientException ex) {
-            Logger.getLogger(Endpoint.class.getName()).log(Level.SEVERE, "Error accessing corpora " + value + " at " + endpoint.getUrl(), ex);
+            logger.log(Level.SEVERE, "Error accessing corpora {0} at {1}\n {2} {3}", 
+                    new String[]{value, endpoint.getUrl(), ex.getClass().getName(), ex.getMessage()});
         }
         if (corporaResponse != null && corporaResponse.hasTerms()) {
@@ -147 +149 @@
         }
     }
+   
 }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/data/Endpoint.java

@@ -23 +23 @@
     private List<Corpus> corpora;
     private boolean hasChildrenLoaded = false;
+    
+    private static final Logger logger = Logger.getLogger(Endpoint.class.getName());
 
     public Endpoint(String url, Institution institution) {
@@ -63 +65 @@
         }
         this.hasChildrenLoaded = true;
-        //this.corpora = EndpointY.getCorpora(url);
         loadChildCorpora();
     }
@@ -80 +81 @@
         }
         return corpora.get(index);
-    }
-
-//    public static ArrayList<CorpusY> getCorpora(String endpointUrl) {
-//        return getCorpora(endpointUrl, ""); //TODO to be changed to 'root'
-//    }
-//
-//    public static ArrayList<CorpusY> getCorpora(String endpointUrl, String resourceHandler) {
-//
-//        ArrayList<CorpusY> corpora = new ArrayList<CorpusY>();
-//        SRUScanResponse corporaResponse = null;
-//        StringBuilder scanClause = new StringBuilder("fcs.resource");
-//        if (resourceHandler.length() > 0) {
-//            scanClause.append("=");
-//        }
-//        scanClause.append(resourceHandler);
-//        try {
-//            SRUClient sruClient = new SRUClient(SRUVersion.VERSION_1_2);
-//            SRUScanRequest corporaRequest = new SRUScanRequest(endpointUrl);
-//            corporaRequest.setScanClause(scanClause.toString());
-//            //TODO extra data?
-//            //corporaRequest.setExtraRequestData("x-cmd-resource-info", "true");
-//            corporaResponse = sruClient.scan(corporaRequest);
-//        } catch (SRUClientException ex) {
-//            Logger.getLogger(EndpointY.class.getName()).log(Level.SEVERE, "Error accessing corpora at " + endpointUrl + " for " + resourceHandler, ex);
-//        }
-//        if (corporaResponse != null && corporaResponse.hasTerms()) {
-//            for (SRUTerm term : corporaResponse.getTerms()) {
-//                CorpusY c = new CorpusY(endpointUrl);
-//                c.setValue(term.getValue());
-//                c.setDisplayTerm(term.getDisplayTerm());
-//                c.setNumberOfRecords(term.getNumberOfRecords());
-//                corpora.add(c);
-//            }
-//        }
-//        return corpora;
-//
-//    }
-    
+    }  
 
     private void loadChildCorpora() {
@@ -132 +96 @@
             corporaResponse = sruClient.scan(corporaRequest);
         } catch (SRUClientException ex) {
-            Logger.getLogger(Endpoint.class.getName()).log(Level.SEVERE, "Error accessing corpora at " + url, ex);
+            logger.log(Level.SEVERE, "Error accessing corpora at {0}\n {1}\n {2}", new String[]{url, ex.getClass().getName(), ex.getMessage()});
         }
         if (corporaResponse != null && corporaResponse.hasTerms()) {

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/data/Institution.java

@@ -3 +3 @@
 import eu.clarin.sru.fcs.aggregator.sparam.CorpusTreeNode;
 import eu.clarin.sru.fcs.aggregator.data.CenterRegistry;
+import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
@@ -24 +25 @@
     private ArrayList<Endpoint> endpoints;
     private boolean hasChildrenLoaded = false;
-    private static final Logger logger = Logger.getLogger("FCS-AGGREGATOR");
+    
+    private static final Logger logger = Logger.getLogger(Institution.class.getName());
 
     public Institution(String name, String link) {
@@ -52 +54 @@
         }
         hasChildrenLoaded = true;
-        InputStream is;
-        URL u;
-        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-        DocumentBuilder builder;
-        Document doc;
-
+        InputStream is = null;
+        
         try {
-            builder = factory.newDocumentBuilder();
-            u = new URL(link);
+            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+            DocumentBuilder builder = factory.newDocumentBuilder();
+            URL u = new URL(link);
             is = u.openStream();
-            doc = builder.parse(is);
+            Document doc = builder.parse(is);
             is.close();
-            NodeList endpointsUrls = CenterRegistry.evaluateXPath("//WebReference[./Description[text()=\"CQL\"]]/Website", doc);
+            NodeList endpointsUrls = CenterRegistry
+                    .evaluateXPath("//WebReference[./Description[text()=\"CQL\"]]/Website", doc);
 
             for (int j = 0; j < endpointsUrls.getLength(); j++) {
@@ -72 +72 @@
             } 
         } catch (Exception ex) {
-            logger.log(Level.SEVERE, String.format("Error accessing endpoints of %s", link), ex);
+            logger.log(Level.SEVERE, "Error accessing endpoint of {0}\n {1}\n {2}", 
+                    new String[]{link, ex.getClass().getName(), ex.getMessage()});
+        } finally {
+            if (is != null) {
+                try {
+                    is.close();
+                } catch (IOException ex) {
+                    logger.log(Level.SEVERE, null, ex);
+                }
+            }
         }
     }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/data/SearchResult.java

@@ -6 +6 @@
 
 import eu.clarin.sru.client.SRUSearchRetrieveResponse;
+import eu.clarin.sru.client.fcs.DataViewKWIC;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.Future;
 import java.util.logging.Level;
@@ -23 +26 @@
     private Future<SRUSearchRetrieveResponse> futureResponse;
     private SRUSearchRetrieveResponse response;
+    private List<DataViewKWIC> dataKWIC = new ArrayList<DataViewKWIC>();
+    
+    private static final Logger logger = Logger.getLogger(SearchResult.class.getName());
+
+    public List<DataViewKWIC> getDataKWIC() {
+        return dataKWIC;
+    }
+
+    public void addKWIC(DataViewKWIC kw) {
+        this.dataKWIC.add(kw);
+    }
+    
 
     public SearchResult(Object nodeData) {
@@ -80 +95 @@
             }
         } catch (Exception ex) {
-            Logger.getLogger(SearchResult.class.getName()).log(Level.SEVERE, null, ex);
+            logger.log(Level.SEVERE, "Error consuming response from {0} {1}\n {2}\n {3}", 
+                    new Object[]{endpoint.getUrl(), corpus, ex.getClass().getName(), ex.getMessage()});
         }
     }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/sresult/SearchResultRecordRenderer.java

@@ -7 +7 @@
 import eu.clarin.sru.client.fcs.DataViewKWIC;
 import eu.clarin.sru.client.fcs.Resource;
+import eu.clarin.sru.fcs.aggregator.data.SearchResult;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.zkoss.zul.Cell;
@@ -21 +23 @@
  */
 public class SearchResultRecordRenderer implements RowRenderer {
+    
+    private SearchResult searchResult;
+    
+    private static final Logger logger = Logger.getLogger(SearchResultRecordRenderer.class.getName());
+    
+    public SearchResultRecordRenderer(SearchResult searchResult) {
+        this.searchResult = searchResult;
+    }
 
     @Override
@@ -27 +37 @@
         SRURecord record = (SRURecord) data;
 
-        Logger.getLogger(this.getClass().getName()).info(
-                String.format("schema = %s, identifier = %s, position = %s",
+        logger.log(Level.FINE,
+                "schema = {0}, identifier = {1}, position = {2}",
                 new Object[]{record.getRecordSchema(),
                     record.getRecordIdentifier(),
-                    record.getRecordPosition()}));
+                    record.getRecordPosition()});
 
         if (record.isRecordSchema(ClarinFCSRecordData.RECORD_SCHEMA)) {
             ClarinFCSRecordData rd =
                     (ClarinFCSRecordData) record.getRecordData();
-
-
-
-
+            
             Resource resource = rd.getResource();
-
+            
+            logger.log(Level.FINE,
+                "Resource ref={0}, pid={1}, dataViews={2}",
+                new Object[]{resource.getRef(), resource.getPid(), resource.hasDataViews()});
 
 //            if (resource.getRef() != null) {
@@ -57 +67 @@
             // If dataviews are assigned directly to the resource:                    
             if (resource.hasDataViews()) {
-                //zeilen.addAll(dataViews2Rows(resource.getDataViews()));
                 appendDataView(row, resource.getDataViews());
             }
@@ -64 +73 @@
             if (resource.hasResourceFragments()) {
                 for (Resource.ResourceFragment fragment : resource.getResourceFragments()) {
-                    Logger.getLogger(this.getClass().getName()).info(
-                            String.format("CLARIN-FCS: ResourceFragment: pid=%s, ref=%s",
-                            fragment.getPid(), fragment.getRef()));
+                    logger.log(Level.FINE, "ResourceFragment: ref={0}, pid={1}, dataViews={2}",
+                            new Object[]{fragment.getRef(), fragment.getPid(), fragment.hasDataViews()});
                     if (fragment.hasDataViews()) {
-                        //zeilen.addAll(dataViews2Rows(fragment.getDataViews()));
                         appendDataView(row, fragment.getDataViews());
                     }
                 }
-            } //ResourceFragments
+            }
 
         } else if (record.isRecordSchema(SRUSurrogateRecordData.RECORD_SCHEMA)) {
             SRUSurrogateRecordData r =
                     (SRUSurrogateRecordData) record.getRecordData();
-            Logger.getLogger(this.getClass().getName()).info(
-                    String.format("SURROGATE DIAGNOSTIC: uri=%s, message=%s, detail=%s",
-                    r.getURI(), r.getMessage(), r.getDetails()));
+            logger.log(Level.INFO, "Surrogate diagnostic: uri={0}, message={1}, detail={2}",
+                    new Object[]{r.getURI(), r.getMessage(), r.getDetails()});
         } else {
-            Logger.getLogger(this.getClass().getName()).info(
-                    String.format("UNSUPPORTED SCHEMA: %s",
-                    record.getRecordSchema()));
+            logger.log(Level.INFO, "Unsupported schema: {0}", record.getRecordSchema());
         }
 
@@ -95 +99 @@
             if (dataview.isMimeType(DataViewKWIC.TYPE)) {
                 DataViewKWIC kw = (DataViewKWIC) dataview;
-
+                this.searchResult.addKWIC(kw);
 
                 Label toTheLeft = new Label();
+                toTheLeft.setValue(kw.getLeft());
+                
                 toTheLeft.setMultiline(true);
-                toTheLeft.setValue(kw.getLeft());
                 toTheLeft.setSclass("word-wrap");
                 Cell toTheLeftCell = new Cell();
@@ -106 +111 @@
                 toTheLeftCell.setValign("bottom");
                 row.appendChild(toTheLeftCell);
-                //row.appendChild(toTheLeft);
+//                row.appendChild(toTheLeft);
                 
                 
@@ -118 +123 @@
                 lCell.setValign("bottom");
                 row.appendChild(lCell);
-                //row.appendChild(l);
+//                row.appendChild(l);
 
                 Label toTheRight = new Label();
+                toTheRight.setValue(kw.getRight());
                 toTheRight.setMultiline(true);
                 toTheRight.setSclass("word-wrap");
-                toTheRight.setValue(kw.getRight());
                 Cell toTheRightCell = new Cell();
                 toTheRightCell.appendChild(toTheRight);
                 toTheRightCell.setValign("bottom");
                 row.appendChild(toTheRightCell);
-                //row.appendChild(toTheRight);
+//                row.appendChild(toTheRight);
 
             }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/sresult/SearchResultsController.java

@@ -9 +9 @@
 import eu.clarin.sru.client.fcs.ClarinFCSRecordData;
 import eu.clarin.sru.client.fcs.ClarinFCSRecordParser;
+import eu.clarin.sru.client.fcs.DataViewKWIC;
+import eu.clarin.sru.fcs.aggregator.app.WebAppListener;
 import eu.clarin.sru.fcs.aggregator.data.Institution;
 import eu.clarin.sru.fcs.aggregator.data.SearchResult;
 import eu.clarin.sru.fcs.aggregator.sparam.CorpusTreeNodeRenderer;
+import eu.clarin.weblicht.wlfxb.io.WLDObjector;
+import eu.clarin.weblicht.wlfxb.io.WLFormatException;
+import eu.clarin.weblicht.wlfxb.md.xb.MetaData;
+import eu.clarin.weblicht.wlfxb.tc.xb.TextCorpusStored;
+import eu.clarin.weblicht.wlfxb.xb.WLData;
+import java.io.ByteArrayOutputStream;
 import java.util.ArrayList;
 import java.util.List;
@@ -18 +26 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.zkoss.zhtml.Filedownload;
 import org.zkoss.zk.ui.Component;
 import org.zkoss.zk.ui.Executions;
@@ -28 +37 @@
 import org.zkoss.zul.Label;
 import org.zkoss.zul.ListModel;
+import org.zkoss.zul.Menuitem;
 import org.zkoss.zul.Messagebox;
 import org.zkoss.zul.SimpleListModel;
@@ -40 +50 @@
 public class SearchResultsController {
 
-    private static final Logger logger = Logger.getLogger("FCS-AGGREGATOR");
     private List<SearchResult> resultsUnprocessed;
     private List<SearchResult> resultsProcessed;
@@ -47 +56 @@
     private UpdateResultsThread resultsThread;
     private int currentRequestId = 0;
-
-    public SearchResultsController(Component resultsArea) {
+    private Label progress;
+    
+    private static final Logger logger = Logger.getLogger(SearchResultsController.class.getName());
+
+    public SearchResultsController(Component resultsArea, Label progress) {
         this.resultsArea = resultsArea;
+        this.progress = progress;
         Executions.getCurrent().getDesktop().enableServerPush(true);
-    }
-
-    public void executeSearch(Set<Treeitem> selectedItems, int maxRecords, String searchString) {
+        searchClient = (SRUThreadedClient) Executions.getCurrent().getDesktop().getWebApp().getAttribute(WebAppListener.SHARED_SRU_CLIENT);
+    }
+
+    public void executeSearch(Set<Treeitem> selectedItems, int maxRecords, String searchString, SRUVersion version) {
 
         // execute search only if a user selected at least one endpint/corpus
@@ -72 +86 @@
         currentRequestId++;
 
-        // initialize client
-        searchClient = new SRUThreadedClient(SRUVersion.VERSION_1_2);
-        try {
-            searchClient.registerRecordParser(new ClarinFCSRecordParser());
-        } catch (SRUClientException e) {
-            logger.log(Level.SEVERE, "SRU Client parser registration failed", e);
-            //TODO error page...
-        }
-
         // clear are where results are to be displayed
         resultsArea.getChildren().clear();
@@ -90 +95 @@
         // finally, send search requests to all the selected by user 
         // endpoints/corpora and process the responses
-        sendRequests(selectedItems, maxRecords, searchString);
+        sendRequests(selectedItems, maxRecords, searchString, version);
         processResponses();
     }
 
-    private void sendRequests(Set<Treeitem> selectedItems, int maxRecords, String searchString) {
-
+    private void sendRequests(Set<Treeitem> selectedItems, int maxRecords, String searchString, SRUVersion version) {
+
+        logger.log(Level.INFO, "Executing query={0} maxRecords={1}", 
+                new Object[]{searchString, maxRecords});
+        
         for (Treeitem selectedItem : selectedItems) {
             Object nodeData = selectedItem.getAttribute(CorpusTreeNodeRenderer.ITEM_DATA);
@@ -102 +110 @@
                 // if there whole parent corpus/endpoint will be queried
             } else {
-                SearchResult resultsItem = executeRequest(nodeData, searchString, maxRecords);
+                SearchResult resultsItem = executeRequest(nodeData, searchString, maxRecords, version);
                 resultsUnprocessed.add(resultsItem);
             }
@@ -108 +116 @@
     }
 
-    private SearchResult executeRequest(Object nodeData, String searchString, int maxRecords) {
+    private SearchResult executeRequest(Object nodeData, String searchString, int maxRecords, SRUVersion version) {
 
         SearchResult resultsItem = new SearchResult(nodeData);
-        //Logger.getLogger("FCS-AGGREGATOR").info("Now executing search: " + searchString + " " + endpoint.getUrl() + " " + corpus.getValue() + " " + maxRecords);
+        logger.log(Level.FINE, "Executing search for {0} query={1} maxRecords={2}", 
+                new Object[]{nodeData.toString(), searchString, maxRecords});
         SRUSearchRetrieveRequest searchRequest = new SRUSearchRetrieveRequest(resultsItem.getEndpoint().getUrl());
+        searchRequest.setVersion(version);
         searchRequest.setMaximumRecords(maxRecords);
         searchRequest.setRecordSchema(ClarinFCSRecordData.RECORD_SCHEMA);
@@ -123 +133 @@
             resultsItem.setFutureResponse(futureResponse);
         } catch (SRUClientException ex) {
-            Logger.getLogger(SearchResultsController.class.getName()).log(Level.INFO, resultsItem.getEndpoint().getUrl(), ex);
+            logger.log(Level.SEVERE, "SearchRetrieve failed for {0}\n {1}\n {2}", 
+                    new String[]{resultsItem.getEndpoint().getUrl(), ex.getClass().getName(), ex.getMessage()});
         }
         return resultsItem;
@@ -130 +141 @@
 
     private void processResponses() {
-        processResponsesAsync();
-        //processResponsesSync();
-
-    }
-
-    private void processResponsesSync() {
-
-        while (!resultsUnprocessed.isEmpty()) {
-            SearchResult resultsItem = resultsUnprocessed.remove(0);
-            if (!resultsItem.isWaitingForResponse()) {
-                resultsItem.consumeResponse();
-                // create groupbox with search results item
-                Groupbox groupbox = createRecordsGroup(resultsItem);
-                // appand this search result only
-                resultsArea.appendChild(groupbox);
-                resultsProcessed.add(resultsItem);
-            } else {
-                resultsUnprocessed.add(resultsItem);
-            }
-
-        }
-    }
-
-    private void processResponsesAsync() {
+        processResponsesWithAsyncResultsWindowUpdate();
+        //processResponsesWithSyncResultsWindowUpdate();
+
+    }
+
+//    private void processResponsesWithSyncResultsWindowUpdate() {
+//
+//        while (!resultsUnprocessed.isEmpty()) {
+//            SearchResult resultsItem = resultsUnprocessed.remove(0);
+//            if (!resultsItem.isWaitingForResponse()) {
+//                resultsItem.consumeResponse();
+//                // create groupbox with search results item
+//                Groupbox groupbox = createRecordsGroup(resultsItem);
+//                // appand this search result only
+//                resultsArea.appendChild(groupbox);
+//                resultsProcessed.add(resultsItem);
+//            } else {
+//                resultsUnprocessed.add(resultsItem);
+//            }
+//
+//        }
+//    }
+    
+    private void processResponsesWithAsyncResultsWindowUpdate() {
         resultsThread = new UpdateResultsThread();
         resultsThread.start();
@@ -163 +174 @@
         public void run() {
             while (!resultsUnprocessed.isEmpty() && !Thread.currentThread().isInterrupted()) {
-                //TODO display progress bar...
+                
                 SearchResult resultsItem = resultsUnprocessed.remove(0);
                 if (!resultsItem.isWaitingForResponse()) {
@@ -182 +193 @@
                     resultsProcessed.add(resultsItem);
                     //System.out.println("RECORDS ITEM ADDED");
+                    
                 } else {
                     resultsUnprocessed.add(resultsItem);
@@ -222 +234 @@
             if (requestId != currentRequestId) {
                 groupbox.detach();
+            }
+            
+            if (resultsUnprocessed.isEmpty()) {
+                progress.setValue("");
+            } else {
+                progress.setValue("waiting for " + resultsUnprocessed.size() + " responses...");
             }
 
@@ -269 +287 @@
             //c.setLabel("Left");
             columns.appendChild(c);
+            //c.setHflex("2");
             c = new Column();
             //c.setLabel("Hit");
             c.setHflex("min");
+            //c.setHflex("1");
             columns.appendChild(c);
             c = new Column();
+            //c.setHflex("2");
             //c.setLabel("Right");
             columns.appendChild(c);
             grid.appendChild(columns);
-            
+
             List<SRURecord> sruRecords = resultsItem.getResponse().getRecords();
             ListModel lmodel = new SimpleListModel(sruRecords);
             grid.setModel(lmodel);
-            grid.setRowRenderer(new SearchResultRecordRenderer());
+            grid.setRowRenderer(new SearchResultRecordRenderer(resultsItem));
             recordsGroup.appendChild(grid);
         } else { // the response was fine, but there are no records
@@ -290 +311 @@
     }
 
+    public void exportCSV() {
+
+        boolean noResult = true;
+        StringBuilder csv = new StringBuilder();
+        if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
+            for (SearchResult result : resultsProcessed) {
+                for (DataViewKWIC kwic : result.getDataKWIC()) {
+                    csv.append("\"");
+                    csv.append(kwic.getLeft().replace("\"", "QUOTE"));
+                    csv.append("\"");
+                    csv.append(",");
+                    csv.append("\"");
+                    csv.append(kwic.getKeyword().replace("\"", "QUOTE"));
+                    csv.append("\"");
+                    csv.append(",");
+                    csv.append("\"");
+                    csv.append(kwic.getRight().replace("\"", "QUOTE"));
+                    csv.append("\"");
+                    csv.append("\n");
+                    noResult = false;
+                }
+            }
+        }
+
+        if (noResult) {
+            Messagebox.show("Nothing to export!");
+        } else {
+            Filedownload.save(csv.toString(), "text/plain", "ClarinDFederatedContentSearch.csv");
+        }
+    }
+
+    public void exportTCF() {
+
+        boolean noResult = true;
+        StringBuilder text = new StringBuilder();
+
+        if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
+            for (SearchResult result : resultsProcessed) {
+                for (DataViewKWIC kwic : result.getDataKWIC()) {
+                    text.append(kwic.getLeft());
+                    text.append(" ");
+                    text.append(kwic.getKeyword());
+                    text.append(" ");
+                    text.append(kwic.getRight());
+                    text.append("\n");
+                    noResult = false;
+                }
+            }
+
+        }
+
+        if (noResult) {
+            Messagebox.show("Nothing to export!");
+        } else {
+            WLData data;
+            MetaData md = new MetaData();
+            //data.metaData.source = "Tuebingen Uni";
+            //md.addMetaDataItem("title", "binding test");
+            //md.addMetaDataItem("author", "Yana");
+            TextCorpusStored tc = new TextCorpusStored("unknown");
+            tc.createTextLayer().addText(text.toString());
+            data = new WLData(md, tc);
+            ByteArrayOutputStream os = new ByteArrayOutputStream();
+            try {
+                WLDObjector.write(data, os);
+                Filedownload.save(os.toByteArray(), "text/tcf+xml", "ClarinDFederatedContentSearch.xml");
+            } catch (WLFormatException ex) {
+                logger.log(Level.SEVERE, "Error exporting TCF {0}\n {1}", new String[]{ex.getClass().getName(), ex.getMessage()});
+                Messagebox.show("Sorry, export error!");
+            }
+        }
+    }
+
     public void shutdown() {
         terminateProcessingRequestsAndResponses();
@@ -295 +389 @@
 
     private void terminateProcessingRequestsAndResponses() {
-
-        if (searchClient != null) {
-            searchClient.shutdown();
-        }
 
         if (resultsThread != null) {
@@ -308 +398 @@
             }
         }
+
         Logger.getLogger(SearchResultsController.class.getName()).log(Level.INFO, "Search terminated");
     }

SRUAggregator/trunk/src/main/webapp/WEB-INF/zk.xml

@@ -15 +15 @@
     </listener>
     <listener>
-        <description>WebAppCleanup cleanup</description>
-        <listener-class>eu.clarin.sru.fcs.aggregator.app.WebAppDestroyedListener</listener-class>
+        <description>WebApp initialization and cleanup</description>
+        <listener-class>eu.clarin.sru.fcs.aggregator.app.WebAppListener</listener-class>
     </listener>
     

SRUAggregator/trunk/src/main/webapp/index.zul

@@ -1 +1 @@
 <?xml version="1.0" encoding="utf-8"?>
 <?page title="CLARIN-D Federated Content Search"?>
-<zk title="CLARIN-D Federated Content Search">
-
+<zk>
     <window id="mainWindow" contentStyle="background:white;" height="100%"
         apply="eu.clarin.sru.fcs.aggregator.app.Aggregator">
@@ -20 +19 @@
                         <hlayout
                             style="margin-top:20px;margin-bottom:10px;margin-left:auto;margin-right:auto;width:95%;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;">
-                            <textbox hflex="1" id="searchString" value="" focus="true"
+                            <textbox hflex="1" id="searchString" value="Elefant" focus="true"
                                 style="font-size:16px;margin:5px 0px 5px 10px;color:#0e4071;border: 1px solid #801418;"/>
 
@@ -163 +162 @@
                         <panel>
                             <panelchildren id="resultsPanel" style="overflow: auto">
-
                                 <window id="resultsBox" style="overflow:auto;">
                                     <iframe src="help.html" width="800px" height="400px"/>
                                 </window>
+                                <label value=" " id="searchResultsProgress" style="border-width:0px;margin-left:5px;"/>
                             </panelchildren>
                         </panel>