Changeset 6065 for SRUAggregator

Timestamp:

02/26/15 16:17:04 (9 years ago)

Author:

emanuel.dima@uni-tuebingen.de

Message:

22. alpha 25: removed unused iso-639-2 support and unused german tokenizer; fixed bug related to xml unencoding; changed weblicht config

Location:

SRUAggregator/trunk

Files:

• aggregator.yml (modified) (2 diffs)
• build.sh (modified) (1 diff)
• pom.xml (modified) (3 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/app/Aggregator.java (modified) (9 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/app/AggregatorConfiguration.java (modified) (3 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/lang (deleted)
• src/main/java/eu/clarin/sru/fcs/aggregator/rest/RestService.java (modified) (8 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/scan/Corpus.java (modified) (2 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/search/Exports.java (modified) (4 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/search/Kwic.java (modified) (3 diffs)
• src/main/java/eu/clarin/sru/fcs/aggregator/util/LanguagesISO693.java (copied) (copied from SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/lang/LanguagesISO693_3.java) (9 diffs)
• src/main/resources/assets/js/main.js (modified) (2 diffs)
• src/main/resources/assets/js/main.jsx (modified) (2 diffs)
• src/main/resources/lang/ISO-639-2_utf-8.txt (deleted)
• src/main/resources/tokenizer (deleted)

SRUAggregator/trunk/aggregator.yml

@@ -1 +1 @@
 aggregatorParams:
-  # CENTER_REGISTRY_URL: http://centres.clarin.eu/restxml/
+  CENTER_REGISTRY_URL: http://centres.clarin.eu/restxml/
   additionalCQLEndpoints:
     - https://clarin.ids-mannheim.de/digibibsru-new
-    - https://lux17.mpi.nl/ds/cqlsearch
+    # - https://lux17.mpi.nl/ds/cqlsearch
     # - http://cqlservlet.mpi.nl/
     # - http://dspin.dwds.de:8088/ddc-sru/dingler/
@@ -28 +28 @@
   EXECUTOR_SHUTDOWN_TIMEOUT_MS: 1000
 
-  WEBLICHT_URL: https://weblicht.sfs.uni-tuebingen.de/WebLicht-4/?input=
+  weblichtConfig:
+    url: https://weblicht.sfs.uni-tuebingen.de/WebLicht-4/?input=
+    acceptedTcfLanguages:
+      - en
+      - de
+      - nl
+      - fr
+      - it
+      - sp
+      - pl
 
 # use the simple server factory, run on a single port

SRUAggregator/trunk/build.sh

@@ -32 +32 @@
 node_modules/react-tools/bin/jsx --no-cache-dir $JSDIR $JSDIR
 
-# mvn -q clean package
+if [ "$1" == "--jar" ]
+then
+        mvn -q clean package
+fi
 
 # Run in production:
-#java -jar target/Aggregator2-2.0.0-alpha-6.jar server aggregator.yml
+#java -jar target/Aggregator2-2.0.0.jar server aggregator.yml
 
 # Run for development:
-# java -cp src/main/resources:target/Aggregator2-2.0.0-alpha-10.jar eu.clarin.sru.fcs.aggregator.app.Aggregator server aggregator_development.yml
+# java -cp src/main/resources:target/Aggregator2-2.0.0.jar eu.clarin.sru.fcs.aggregator.app.Aggregator server aggregator_development.yml

SRUAggregator/trunk/pom.xml

@@ -8 +8 @@
         <groupId>eu.clarin.sru.fcs</groupId>
         <artifactId>Aggregator2</artifactId>
-        <version>2.0.0-alpha-24</version>
+        <version>2.0.0-alpha-25</version>
         <name>FCS Aggregator</name>
 
@@ -90 +90 @@
 
                 <dependency>
-                        <groupId>org.apache.opennlp</groupId>
-                        <artifactId>opennlp-tools</artifactId>
-                        <version>1.5.3</version>
-                </dependency>
-                <dependency>
                     <groupId>com.optimaize.languagedetector</groupId>
                     <artifactId>language-detector</artifactId>
@@ -149 +144 @@
                                 </configuration>
                         </plugin>
-                        <!--<plugin>
-                                <groupId>org.apache.maven.plugins</groupId>
-                                <artifactId>maven-source-plugin</artifactId>
-                                <version>2.2.1</version>
-                                <executions>
-                                        <execution>
-                                                <id>attach-sources</id>
-                                                <goals>
-                                                        <goal>jar</goal>
-                                                </goals>
-                                        </execution>
-                                </executions>
-                        </plugin>-->
                         <plugin>
                                 <groupId>org.apache.maven.plugins</groupId>

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/Aggregator.java

@@ -18 +18 @@
 import eu.clarin.sru.fcs.aggregator.rest.RestService;
 import eu.clarin.sru.fcs.aggregator.scan.Statistics;
-import eu.clarin.sru.fcs.aggregator.lang.LanguagesISO693_3;
+import eu.clarin.sru.fcs.aggregator.util.LanguagesISO693;
 import io.dropwizard.Application;
 import io.dropwizard.assets.AssetsBundle;
@@ -25 +25 @@
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -35 +34 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicReference;
-import opennlp.tools.tokenize.TokenizerModel;
 import org.slf4j.LoggerFactory;
 
@@ -88 +86 @@
  * @author edima
  *
+ * TODO: ?use weblicht only to show up in zoomed mode
+ * - send only tcf with only a text layer and language (from the list in params)
+ *
+ * TODO: add the modes described above (except live)
+ *
  * TODO: zoom into the results from a corpus, allow functionality only for
  * the view (search for next set of results)
@@ -95 +98 @@
  * Twan (they did a test, it worked)
  *
+ * TODO: add PiWik support, tracking the following:
+ * - visits, searches, search per corpus
+ *
+ * TODO: BUG: language detection is immediate, in UI; export implications
+ *
  * TODO: websockets
  *
@@ -122 +130 @@
         private AtomicReference<Statistics> searchStatsAtom = new AtomicReference<Statistics>(new Statistics());
 
-        private TokenizerModel tokenizerModel;
         private LanguageDetector languageDetector;
         private TextObjectFactory textObjectFactory;
@@ -235 +242 @@
                 }
 
-                LanguagesISO693_3.getInstance(); // force init
-                initTokenizer();
+                LanguagesISO693.getInstance(); // force init
                 initLanguageDetector();
 
@@ -293 +299 @@
         }
 
-        public TokenizerModel getTokenizerModel() {
-                return tokenizerModel;
-        }
-
         private static void shutdownAndAwaitTermination(AggregatorConfiguration.Params params,
                         ThrottledClient sruClient, ExecutorService scheduler) {
@@ -313 +315 @@
         }
 
-        private void initTokenizer() {
-                TokenizerModel model = null;
-                try {
-                        try (InputStream tokenizerModelDeAsIS = Thread.currentThread().getContextClassLoader().getResourceAsStream(DE_TOK_MODEL)) {
-                                model = new TokenizerModel(tokenizerModelDeAsIS);
-                        }
-                } catch (IOException ex) {
-                        log.error("Failed to load tokenizer model", ex);
-                }
-                tokenizerModel = model;
-        }
-
         public void initLanguageDetector() throws IOException {
                 List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAll();

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/AggregatorConfiguration.java

@@ -14 +14 @@
         public static class Params {
 
-                @NotEmpty
                 @JsonProperty
                 String CENTER_REGISTRY_URL;
 
-                @NotEmpty
                 @JsonProperty
-                String WEBLICHT_URL;
+                List<URL> additionalCQLEndpoints;
 
                 @NotEmpty
@@ -66 +64 @@
                 long EXECUTOR_SHUTDOWN_TIMEOUT_MS;
 
+                public static class WeblichtConfig {
+                        @JsonProperty
+                        String url;
+
+                        @JsonProperty
+                        List<String> acceptedTcfLanguages;
+
+                        @JsonIgnore
+                        public String getUrl() {
+                                return url;
+                        }
+
+                        @JsonIgnore
+                        public List<String> getAcceptedTcfLanguages() {
+                                return acceptedTcfLanguages;
+                        }
+                }
+
+                @NotEmpty
                 @JsonProperty
-                List<URL> additionalCQLEndpoints;
+                WeblichtConfig weblichtConfig;
 
                 @JsonIgnore
@@ -95 +112 @@
 
                 @JsonIgnore
-                public String getWEBLICHT_URL() {
-                        return WEBLICHT_URL;
+                public WeblichtConfig getWeblichtConfig() {
+                        return weblichtConfig;
                 }
         }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/rest/RestService.java

@@ -7 +7 @@
 import eu.clarin.sru.fcs.aggregator.app.Aggregator;
 import eu.clarin.sru.fcs.aggregator.app.AggregatorConfiguration;
+import eu.clarin.sru.fcs.aggregator.app.AggregatorConfiguration.Params.WeblichtConfig;
 import eu.clarin.sru.fcs.aggregator.scan.Corpus;
 import eu.clarin.sru.fcs.aggregator.scan.Statistics;
@@ -12 +13 @@
 import eu.clarin.sru.fcs.aggregator.search.Result;
 import eu.clarin.sru.fcs.aggregator.search.Search;
-import eu.clarin.sru.fcs.aggregator.lang.LanguagesISO693_3;
-import eu.clarin.sru.fcs.aggregator.search.ExportException;
+import eu.clarin.sru.fcs.aggregator.util.LanguagesISO693;
 import eu.clarin.sru.fcs.aggregator.search.Exports;
 import java.io.IOException;
@@ -22 +22 @@
 import java.util.Map;
 import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 import javax.servlet.ServletContext;
 import javax.servlet.http.HttpServletRequest;
@@ -36 +34 @@
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-import opennlp.tools.tokenize.TokenizerModel;
 import org.slf4j.LoggerFactory;
 
@@ -113 +110 @@
                 log.info("get language codes", codes);
                 for (String code : codes) {
-                        String name = LanguagesISO693_3.getInstance().nameForCode_3(code);
+                        String name = LanguagesISO693.getInstance().nameForCode(code);
                         languages.put(code, name != null ? name : code);
                 }
@@ -192 +189 @@
                 }
 
-        if (format == null || format.trim().isEmpty() || format.trim().equals("text")) {
+                if (format == null || format.trim().isEmpty() || format.trim().equals("text")) {
                         String text = Exports.getExportText(search.getResults());
                         return download(text, MediaType.TEXT_PLAIN, search.getQuery() + ".txt");
                 } else if (format.equals("tcf")) {
-                        byte[] bytes = Exports.getExportTokenizedTCF(
-                                        search.getResults(), search.getSearchLanguage(),
-                                        Aggregator.getInstance().getTokenizerModel());
+                        byte[] bytes = Exports.getExportTCF(
+                                        search.getResults(), search.getSearchLanguage());
                         return download(bytes, TCF_MEDIA_TYPE, search.getQuery() + ".xml");
                 } else if (format.equals("excel")) {
@@ -240 +236 @@
                                 url = DataTransfer.uploadToDropOff(bytes, "text/plan", ".txt");
                         }
-                } else if (format.equals("tokens")) {
-                        byte[] bytes = Exports.getExportTokenizedTCF(
-                                        search.getResults(), search.getSearchLanguage(),
-                                        Aggregator.getInstance().getTokenizerModel());
+                } else if (format.equals("tcf")) {
+                        byte[] bytes = Exports.getExportTCF(
+                                        search.getResults(), search.getSearchLanguage());
                         if (bytes != null) {
                                 url = DataTransfer.uploadToDropOff(bytes, "text/tcf+xml", ".tcf");
@@ -251 +246 @@
                 }
 
-                URI weblichtUri = new URI(Aggregator.getInstance().getParams().getWEBLICHT_URL()
-                                + url);
+                WeblichtConfig weblicht = Aggregator.getInstance().getParams().getWeblichtConfig();
+                URI weblichtUri = new URI(weblicht.getUrl() + url);
                 return url == null
                                 ? Response.status(503).entity("error while exporting to weblicht").build()

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/scan/Corpus.java

@@ -1 +1 @@
 package eu.clarin.sru.fcs.aggregator.scan;
 
-import eu.clarin.sru.fcs.aggregator.lang.LanguagesISO693_3;
+import eu.clarin.sru.fcs.aggregator.util.LanguagesISO693;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -102 +102 @@
 
         public void addLanguage(String language) {
-                if (LanguagesISO693_3.getInstance().getCodes_3().contains(language)) {
+                if (LanguagesISO693.getInstance().isCode(language)) {
                         this.languages.add(language);
                 } else {
-                        String code = LanguagesISO693_3.getInstance().code_3ForName(language);
+                        String code = LanguagesISO693.getInstance().code_3ForName(language);
                         this.languages.add(code == null ? language : code);
                 }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/search/Exports.java

@@ -1 +1 @@
 package eu.clarin.sru.fcs.aggregator.search;
 
-import eu.clarin.sru.fcs.aggregator.lang.LanguagesISO693_2;
-import eu.clarin.sru.fcs.aggregator.lang.LanguagesISO693_3;
+import eu.clarin.sru.fcs.aggregator.util.LanguagesISO693;
 import eu.clarin.weblicht.wlfxb.io.WLDObjector;
 import eu.clarin.weblicht.wlfxb.io.WLFormatException;
 import eu.clarin.weblicht.wlfxb.md.xb.MetaData;
-import eu.clarin.weblicht.wlfxb.tc.api.MatchedCorpus;
-import eu.clarin.weblicht.wlfxb.tc.api.Token;
 import eu.clarin.weblicht.wlfxb.tc.xb.TextCorpusStored;
 import eu.clarin.weblicht.wlfxb.xb.WLData;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import opennlp.tools.tokenize.TokenizerME;
-import opennlp.tools.tokenize.TokenizerModel;
 import org.apache.poi.ss.usermodel.Cell;
 import org.apache.poi.ss.usermodel.CellStyle;
@@ -30 +21 @@
 /**
  * Utility for representing SearchResult data in different formats.
- * 
+ *
  * @author Yana Panchenko
  */
@@ -39 +30 @@
         public static String getExportCSV(List<Result> resultsProcessed, String separator) {
 
-        boolean noResult = true;
-        StringBuilder csv = new StringBuilder();
-        if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
-            String[] headers = new String[]{
-                "LEFT CONTEXT", "KEYWORD", "RIGHT CONTEXT", "PID", "REFERENCE"};
-            for (String header : headers) {
-                csv.append("\"");
-                csv.append(header);
-                csv.append("\"");
-                csv.append(separator);
-            }
-            csv.append("\n");
-
-            for (Result result : resultsProcessed) {
-                for (Kwic kwic : result.getKwics()) {
-                    csv.append("\"");
-                    csv.append(escapeQuotes(kwic.getLeft()));
-                    csv.append("\"");
-                    csv.append(separator);
-                    csv.append("\"");
-                    csv.append(escapeQuotes(kwic.getKeyword()));
-                    csv.append("\"");
-                    csv.append(separator);
-                    csv.append("\"");
-                    csv.append(escapeQuotes(kwic.getRight()));
-                    csv.append("\"");
-                    csv.append(separator);
-                    csv.append("\"");
-                    if (kwic.getPid() != null) {
-                        csv.append(escapeQuotes(kwic.getPid()));
-                    }
-                    csv.append("\"");
-                    csv.append(separator);
-                    csv.append("\"");
-                    if (kwic.getReference() != null) {
-                        csv.append(escapeQuotes(kwic.getReference()));
-                    }
-                    csv.append("\"");
-                    csv.append("\n");
-                    noResult = false;
-                }
-            }
-        }
-        if (noResult) {
-            return null;
-        } else {
-            return csv.toString();
-        }
-    }
+                boolean noResult = true;
+                StringBuilder csv = new StringBuilder();
+                if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
+                        String[] headers = new String[]{
+                                "LEFT CONTEXT", "KEYWORD", "RIGHT CONTEXT", "PID", "REFERENCE"};
+                        for (String header : headers) {
+                                csv.append("\"");
+                                csv.append(header);
+                                csv.append("\"");
+                                csv.append(separator);
+                        }
+                        csv.append("\n");
+
+                        for (Result result : resultsProcessed) {
+                                for (Kwic kwic : result.getKwics()) {
+                                        csv.append("\"");
+                                        csv.append(escapeQuotes(kwic.getLeft()));
+                                        csv.append("\"");
+                                        csv.append(separator);
+                                        csv.append("\"");
+                                        csv.append(escapeQuotes(kwic.getKeyword()));
+                                        csv.append("\"");
+                                        csv.append(separator);
+                                        csv.append("\"");
+                                        csv.append(escapeQuotes(kwic.getRight()));
+                                        csv.append("\"");
+                                        csv.append(separator);
+                                        csv.append("\"");
+                                        if (kwic.getPid() != null) {
+                                                csv.append(escapeQuotes(kwic.getPid()));
+                                        }
+                                        csv.append("\"");
+                                        csv.append(separator);
+                                        csv.append("\"");
+                                        if (kwic.getReference() != null) {
+                                                csv.append(escapeQuotes(kwic.getReference()));
+                                        }
+                                        csv.append("\"");
+                                        csv.append("\n");
+                                        noResult = false;
+                                }
+                        }
+                }
+                if (noResult) {
+                        return null;
+                } else {
+                        return csv.toString();
+                }
+        }
 
         private static CharSequence escapeQuotes(String text) {
-        StringBuilder sb = new StringBuilder();
-        for (int i = 0; i < text.length(); i++) {
-            char ch = text.charAt(i);
-            if (ch == '"') {
-                sb.append('"');
-            }
-            sb.append(ch);
-        }
-        return sb;
-    }
+                StringBuilder sb = new StringBuilder();
+                for (int i = 0; i < text.length(); i++) {
+                        char ch = text.charAt(i);
+                        if (ch == '"') {
+                                sb.append('"');
+                        }
+                        sb.append(ch);
+                }
+                return sb;
+        }
 
         public static byte[] getExportExcel(List<Result> resultsProcessed) throws ExportException {
 
-        boolean noResult = true;
-        SXSSFWorkbook workbook = null;
-        ByteArrayOutputStream excelStream = new ByteArrayOutputStream();
-        if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
-            try {
-                String[] headers = new String[]{
-                    "LEFT CONTEXT", "KEYWORD", "RIGHT CONTEXT", "PID", "REFERENCE"};
-
-                workbook = new SXSSFWorkbook();
-                Sheet sheet = workbook.createSheet();
-
-                Font boldFont = workbook.createFont();
-                boldFont.setBoldweight(Font.BOLDWEIGHT_BOLD);
-
-                // Header
-                CellStyle headerStyle = workbook.createCellStyle();
-                headerStyle.setFont(boldFont);
-
-                Row row = sheet.createRow(0);
-
-                for (int j = 0; j < headers.length; ++j) {
-                    Cell cell = row.createCell(j, Cell.CELL_TYPE_STRING);
-                    cell.setCellValue(headers[j]);
-                    cell.setCellStyle(headerStyle);
-                }
-
-                // Body
-                Cell cell;
-                for (int k = 0; k < resultsProcessed.size(); k++) {
-                    Result result = resultsProcessed.get(k);
-                    List<Kwic> kwics = result.getKwics();
-                    for (int i = 0; i < kwics.size(); i++) {
-                        Kwic kwic = kwics.get(i);
-                        row = sheet.createRow(k + i + 1);
-                        cell = row.createCell(0, Cell.CELL_TYPE_STRING);
-                        cell.setCellValue(kwic.getLeft());
-                        cell = row.createCell(1, Cell.CELL_TYPE_STRING);
-                        cell.setCellValue(kwic.getKeyword());
-                        cell = row.createCell(2, Cell.CELL_TYPE_STRING);
-                        cell.setCellValue(kwic.getRight());
-                        if (kwic.getPid() != null) {
-                            cell = row.createCell(3, Cell.CELL_TYPE_STRING);
-                            cell.setCellValue(kwic.getPid());
-                        }
-                        if (kwic.getReference() != null) {
-                            cell = row.createCell(3, Cell.CELL_TYPE_STRING);
-                            cell.setCellValue(kwic.getReference());
-                        }
-                        noResult = false;
-                    }
-                }
-                workbook.write(excelStream);
-            } catch (IOException ex) {
+                boolean noResult = true;
+                SXSSFWorkbook workbook = null;
+                ByteArrayOutputStream excelStream = new ByteArrayOutputStream();
+                if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
+                        try {
+                                String[] headers = new String[]{
+                                        "LEFT CONTEXT", "KEYWORD", "RIGHT CONTEXT", "PID", "REFERENCE"};
+
+                                workbook = new SXSSFWorkbook();
+                                Sheet sheet = workbook.createSheet();
+
+                                Font boldFont = workbook.createFont();
+                                boldFont.setBoldweight(Font.BOLDWEIGHT_BOLD);
+
+                                // Header
+                                CellStyle headerStyle = workbook.createCellStyle();
+                                headerStyle.setFont(boldFont);
+
+                                Row row = sheet.createRow(0);
+
+                                for (int j = 0; j < headers.length; ++j) {
+                                        Cell cell = row.createCell(j, Cell.CELL_TYPE_STRING);
+                                        cell.setCellValue(headers[j]);
+                                        cell.setCellStyle(headerStyle);
+                                }
+
+                                // Body
+                                Cell cell;
+                                for (int k = 0; k < resultsProcessed.size(); k++) {
+                                        Result result = resultsProcessed.get(k);
+                                        List<Kwic> kwics = result.getKwics();
+                                        for (int i = 0; i < kwics.size(); i++) {
+                                                Kwic kwic = kwics.get(i);
+                                                row = sheet.createRow(k + i + 1);
+                                                cell = row.createCell(0, Cell.CELL_TYPE_STRING);
+                                                cell.setCellValue(kwic.getLeft());
+                                                cell = row.createCell(1, Cell.CELL_TYPE_STRING);
+                                                cell.setCellValue(kwic.getKeyword());
+                                                cell = row.createCell(2, Cell.CELL_TYPE_STRING);
+                                                cell.setCellValue(kwic.getRight());
+                                                if (kwic.getPid() != null) {
+                                                        cell = row.createCell(3, Cell.CELL_TYPE_STRING);
+                                                        cell.setCellValue(kwic.getPid());
+                                                }
+                                                if (kwic.getReference() != null) {
+                                                        cell = row.createCell(3, Cell.CELL_TYPE_STRING);
+                                                        cell.setCellValue(kwic.getReference());
+                                                }
+                                                noResult = false;
+                                        }
+                                }
+                                workbook.write(excelStream);
+                        } catch (IOException ex) {
                                 LOGGER.log(Level.SEVERE, null, ex);
                                 throw new ExportException("Exception exporting Excel", ex);
-            } finally {
-                if (workbook != null) {
-                    workbook.dispose();
-                }
-            }
-        }
-        if (noResult) {
-            return null;
-        } else {
-            return excelStream.toByteArray();
-        }
-
-    }
-
-        private static byte[] getExportTCF(List<Result> resultsProcessed,
+                        } finally {
+                                if (workbook != null) {
+                                        workbook.dispose();
+                                }
+                        }
+                }
+                if (noResult) {
+                        return null;
+                } else {
+                        return excelStream.toByteArray();
+                }
+
+        }
+
+        public static byte[] getExportTCF(List<Result> resultsProcessed,
                         String searchLanguage) throws ExportException {
-        StringBuilder text = new StringBuilder();
-        Set<String> resultsLangs = new HashSet<String>();
-        if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
-            for (Result result : resultsProcessed) {
-                resultsLangs.addAll(result.getCorpus().getLanguages());
-                for (Kwic kwic : result.getKwics()) {
-                    text.append(kwic.getLeft());
-                    text.append(" ");
-                    text.append(kwic.getKeyword());
-                    text.append(" ");
-                    text.append(kwic.getRight());
-                    text.append("\n");
-                }
-            }
-
-        }
-        if (text.length() == 0) {
-                        return null;
-        } else {
-            WLData data;
-            MetaData md = new MetaData();
-            String resultsLang = "unknown";
-            if (resultsLangs.size() == 1) {
-                resultsLang = resultsLangs.iterator().next();
-                                String code2 = LanguagesISO693_2.getInstance().langForCode(resultsLang).getCode_639_1();
-                if (code2 != null) {
-                    resultsLang = code2;
-                }
-            } else if (!searchLanguage.equals("anylang")) {
-                                String code2 = LanguagesISO693_2.getInstance().langForCode(resultsLang).getCode_639_1();
-                if (code2 == null) {
-                    resultsLang = searchLanguage;
-                } else {
-                    resultsLang = code2;
-                }
-            }
-            TextCorpusStored tc = new TextCorpusStored(resultsLang);
-            tc.createTextLayer().addText(text.toString());
-            data = new WLData(md, tc);
+                String text = getExportText(resultsProcessed);
+                if (text == null || text.isEmpty()) {
+                        return null;
+                } else {
+                        WLData data;
+                        MetaData md = new MetaData();
+                        String languageCode = LanguagesISO693.getInstance().code_1ForCode_3(searchLanguage);
+                        TextCorpusStored tc = new TextCorpusStored(languageCode);
+                        tc.createTextLayer().addText(text);
+                        data = new WLData(md, tc);
                         ByteArrayOutputStream os = new ByteArrayOutputStream();
-            try {
-                WLDObjector.write(data, os);
-            } catch (WLFormatException ex) {
-                LOGGER.log(Level.SEVERE, "Error exporting TCF {0} {1}", new String[]{ex.getClass().getName(), ex.getMessage()});
+                        try {
+                                WLDObjector.write(data, os);
+                        } catch (WLFormatException ex) {
+                                LOGGER.log(Level.SEVERE, "Error exporting TCF {0} {1}", new String[]{ex.getClass().getName(), ex.getMessage()});
                                 throw new ExportException("Error exporting TCF", ex);
-            }
+                        }
                         return os.toByteArray();
-        }
-    }
-
-        public static byte[] getExportTokenizedTCF(List<Result> resultsProcessed,
-                        String searchLanguage, TokenizerModel tokenizerModel) throws ExportException {
-        StringBuilder text = new StringBuilder();
-        if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
+                }
+        }
+
+        public static String getExportText(List<Result> resultsProcessed) {
+                StringBuilder text = new StringBuilder();
+                if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
                         for (Result result : resultsProcessed) {
                                 for (Kwic kwic : result.getKwics()) {
@@ -239 +200 @@
                                         }
                                         text.append("\n");
-                }
-            }
-
-        }
-        if (text.length() == 0) {
-                        return null;
-        } else {
-            WLData data;
-                        MetaData md = new MetaData();
-                        String languageCode = LanguagesISO693_3.getInstance().code_1ForCode_3(searchLanguage);
-                        TextCorpusStored tc = new TextCorpusStored(languageCode);
-            tc.createTextLayer().addText(text.toString());
-                        addTokensSentencesMatches(resultsProcessed, tc, tokenizerModel);
-            data = new WLData(md, tc);
-                        ByteArrayOutputStream os = new ByteArrayOutputStream();
-            try {
-                WLDObjector.write(data, os);
-            } catch (WLFormatException ex) {
-                LOGGER.log(Level.SEVERE, "Error exporting TCF {0} {1}", new String[]{ex.getClass().getName(), ex.getMessage()});
-                                throw new ExportException("Error exporting TCF", ex);
-                        }
-                        return os.toByteArray();
-                }
-    }
-
-        private static void addTokensSentencesMatches(List<Result> resultsProcessed, TextCorpusStored tc, TokenizerModel model) {
-                if (model == null || !"de".equals(tc.getLanguage())) {
-            return;
-        }
-        TokenizerME tokenizer = new TokenizerME(model);
-
-        if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
-            tc.createTokensLayer();
-            tc.createSentencesLayer();
-            tc.createMatchesLayer("FCS", resultsProcessed.get(0).getSearchString());
-            for (Result result : resultsProcessed) {
-                                MatchedCorpus mCorpus = tc.getMatchesLayer().addCorpus(result.getCorpus().getTitle(), result.getCorpus().getHandle());
-                for (Kwic kwic : result.getKwics()) {
-                    List<Token> tokens = new ArrayList<Token>();
-                    addToTcfTokens(tokens, tc, tokenizer.tokenize(kwic.getLeft()));
-                    String[] target = tokenizer.tokenize(kwic.getKeyword());
-                    List<Token> targetTokens = addToTcfTokens(tokens, tc, target);
-                    addToTcfTokens(tokens, tc, tokenizer.tokenize(kwic.getRight()));
-                    tc.getSentencesLayer().addSentence(tokens);
-                    List<String> pidAndRef = new ArrayList<String>();
-                    if (kwic.getPid() != null) {
-                        pidAndRef.add(kwic.getPid());
-                    }
-                    if (kwic.getReference() != null) {
-                        pidAndRef.add(kwic.getReference());
-                    }
-                    tc.getMatchesLayer().addItem(mCorpus, targetTokens, pidAndRef);
-                }
-            }
-        }
-    }
-
-        private static List<Token> addToTcfTokens(List<Token> tokens, TextCorpusStored tc, String[] tokenStrings) {
-        List<Token> addedTokens = new ArrayList<Token>(tokenStrings.length);
-        for (String tokenString : tokenStrings) {
-            Token token = tc.getTokensLayer().addToken(tokenString);
-            addedTokens.add(token);
-            tokens.add(token);
-        }
-        return addedTokens;
-    }
-
-        public static String getExportText(List<Result> resultsProcessed) {
-        StringBuilder text = new StringBuilder();
-        if (resultsProcessed != null && !resultsProcessed.isEmpty()) {
-            for (Result result : resultsProcessed) {
-                for (Kwic kwic : result.getKwics()) {
-                    text.append(kwic.getLeft());
-                    text.append(" ");
-                    text.append(kwic.getKeyword());
-                    text.append(" ");
-                    text.append(kwic.getRight());
-                    text.append("\n");
-                }
-            }
-
-        }
-        if (text.length() == 0) {
-            return null;
-        } else {
-            return text.toString();
-        }
-    }
+                                }
+                        }
+                }
+                if (text.length() == 0) {
+                        return null;
+                } else {
+                        return text.toString();
+                }
+        }
 }

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/search/Kwic.java

@@ -3 +3 @@
 import eu.clarin.sru.client.fcs.DataViewHits;
 import eu.clarin.sru.fcs.aggregator.app.Aggregator;
-import eu.clarin.sru.fcs.aggregator.lang.LanguagesISO693_3;
+import eu.clarin.sru.fcs.aggregator.util.LanguagesISO693;
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.commons.lang.StringEscapeUtils;
 
 /**
@@ -48 +49 @@
                 this.reference = reference;
 
-                String text = hits.getText();
+                // warning: the client library doesn't unescape the xml
+                // so the text can still contains < and & codes
+                String str = hits.getText();
+
                 int lastOffset = 0;
                 for (int i = 0; i < hits.getHitCount(); i++) {
                         int[] offsets = hits.getHitOffsets(i);
                         if (lastOffset < offsets[0]) {
-                                fragments.add(new TextFragment(text.substring(lastOffset, offsets[0]), false));
+                                String text = StringEscapeUtils.unescapeXml(str.substring(lastOffset, offsets[0]));
+                                fragments.add(new TextFragment(text, false));
                         }
                         if (offsets[0] < offsets[1]) {
-                                fragments.add(new TextFragment(text.substring(offsets[0], offsets[1]), true));
+                                String text = StringEscapeUtils.unescapeXml(str.substring(offsets[0], offsets[1]));
+                                fragments.add(new TextFragment(text, true));
                         }
                         lastOffset = offsets[1];
                 }
-                if (lastOffset < text.length()) {
-                        fragments.add(new TextFragment(text.substring(lastOffset, text.length()), false));
+                if (lastOffset < str.length()) {
+                        String text = StringEscapeUtils.unescapeXml(str.substring(lastOffset, str.length()));
+                        fragments.add(new TextFragment(text, false));
                 }
 
-                String code_iso639_1 = Aggregator.getInstance().detectLanguage(hits.getText());
+                String code_iso639_1 = Aggregator.getInstance().detectLanguage(str);
                 language = code_iso639_1 == null ? null
-                                : LanguagesISO693_3.getInstance().code_3ForCode_1(code_iso639_1);
+                                : LanguagesISO693.getInstance().code_3ForCode(code_iso639_1);
         }
 
@@ -87 +94 @@
         @Deprecated
         public String getLeft() {
+                StringBuilder sb = new StringBuilder();
                 for (TextFragment tf : fragments) {
-                        if (!tf.isHit) {
-                                return tf.text;
+                        if (tf.isHit) {
+                                break;
                         }
+                        sb.append(tf.text);
                 }
-                return "";
+                return sb.toString();
         }
 

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/util/LanguagesISO693.java

@@ -1 @@
-package eu.clarin.sru.fcs.aggregator.lang;
+package eu.clarin.sru.fcs.aggregator.util;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -10 +10 @@
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Set;
 import org.slf4j.LoggerFactory;
 
@@ -18 +17 @@
  * @author Yana Panchenko
  */
-public class LanguagesISO693_3 {
+public class LanguagesISO693 {
 
-        private static final org.slf4j.Logger log = LoggerFactory.getLogger(LanguagesISO693_3.class);
+        private static final org.slf4j.Logger log = LoggerFactory.getLogger(LanguagesISO693.class);
         public static final String LANGUAGES_FILE_PATH = "/lang/iso-639-3_20140320.tab";
         public static final String LANGUAGES_FILE_ENCODING = "UTF-8";
 
-        private static LanguagesISO693_3 instance = null;
+        private static LanguagesISO693 instance = null;
 
         public static class Language {
 
                 // code is ISO-639-3 (3 letters) while code_2 is ISO-639-1 (2 letters)
-                String code_3, code_1, name;
+                private String code_3, code_1, name;
 
                 public Language(String code_3, String code_1, String name) {
@@ -38 +37 @@
         }
 
-        private Map<String, Language> code_3ToLang = new HashMap<String, Language>();
+        private Map<String, Language> codeToLang = new HashMap<String, Language>();
         private Map<String, Language> nameToLang = new HashMap<String, Language>();
-        private Map<String, Language> code_1ToLang = new HashMap<String, Language>();
 
-        private LanguagesISO693_3() {
-                InputStream is = LanguagesISO693_3.class.getResourceAsStream(LANGUAGES_FILE_PATH);
+        private LanguagesISO693() {
+                InputStream is = LanguagesISO693.class.getResourceAsStream(LANGUAGES_FILE_PATH);
                 try (BufferedReader br = new BufferedReader(new InputStreamReader(is, LANGUAGES_FILE_ENCODING))) {
-                        String line = br.readLine(); // ignore first line
+                        br.readLine(); // ignore first line (header)
+                        String line;
                         while ((line = br.readLine()) != null) {
                                 if (line.length() > 0) {
@@ -60 +59 @@
                                         String name = toks[6].trim();
                                         Language l = new Language(code_3, code_1, name);
-                                        code_3ToLang.put(code_3, l);
+                                        codeToLang.put(code_3, l);
                                         if (code_1 != null) {
-                                                code_1ToLang.put(code_1, l);
+                                                codeToLang.put(code_1, l);
                                         }
                                         nameToLang.put(name, l);
@@ -73 +72 @@
                 ObjectWriter ow = new ObjectMapper().writerWithDefaultPrettyPrinter();
                 try {
-                        System.out.println(ow.writeValueAsString(code_3ToLang));
+                        System.out.println(ow.writeValueAsString(codeToLang));
                 } catch (JsonProcessingException ex) {
                 }
         }
 
-        public static LanguagesISO693_3 getInstance() {
+        public static LanguagesISO693 getInstance() {
                 if (instance == null) {
-                        instance = new LanguagesISO693_3();
+                        instance = new LanguagesISO693();
                 }
                 return instance;
         }
 
-        public Set<String> getCodes_3() {
-                return code_3ToLang.keySet();
+        public boolean isCode(String code) {
+                return codeToLang.containsKey(code);
         }
 
-        public String code_3ForCode_1(String code639_1) {
+        public String code_3ForCode(String code639_1) {
                 if (code639_1 == null) {
                         return null;
                 }
-                Language l = code_1ToLang.get(code639_1);
+                Language l = codeToLang.get(code639_1);
                 if (l == null) {
                         log.error("Unknown ISO-639-1 code: " + code639_1);
@@ -105 +104 @@
                         return null;
                 }
-                Language l = code_3ToLang.get(code639_3);
+                Language l = codeToLang.get(code639_3);
                 if (l == null) {
                         log.error("Unknown ISO-639-3 code: " + code639_3);
@@ -122 +121 @@
         }
 
-        public String nameForCode_3(String code) {
-                Language l = code_3ToLang.get(code);
+        public String nameForCode(String code) {
+                Language l = codeToLang.get(code);
                 if (l == null) {
                         log.error("Unknown language code: " + code);
@@ -130 +129 @@
                 return l.name;
         }
-
 }

SRUAggregator/trunk/src/main/resources/assets/js/main.js

@@ -3 +3 @@
 "use strict";
 
-var VERSION = "VERSION 2.0.0.α24";
+var VERSION = "VERSION 2.0.0.α25";
 var URLROOT = "/Aggregator-testing";
 
@@ -150 +150 @@
                                                                 React.createElement("span", {className: "icon-bar"})
                                                         ), 
-                                                        React.createElement("a", {className: "navbar-brand", href: "#", tabIndex: "-1"}, React.createElement("header", null, "Federated Content Search"))
+                                                        React.createElement("a", {className: "navbar-brand", href: URLROOT, tabIndex: "-1"}, React.createElement("header", null, "Federated Content Search"))
                                                 ), 
                                                 this.renderCollapsible()

SRUAggregator/trunk/src/main/resources/assets/js/main.jsx

@@ -3 +3 @@
 "use strict";
 
-var VERSION = "VERSION 2.0.0.α24";
+var VERSION = "VERSION 2.0.0.α25";
 var URLROOT = "/Aggregator-testing";
 
@@ -150 +150 @@
                                                                 <span className="icon-bar"></span>
                                                         </button>
-                                                        <a className="navbar-brand" href="#" tabIndex="-1"><header>Federated Content Search</header></a>
+                                                        <a className="navbar-brand" href={URLROOT} tabIndex="-1"><header>Federated Content Search</header></a>
                                                 </div>
                                                 {this.renderCollapsible()}