Changeset 6065 for SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/Aggregator.java

Timestamp:

02/26/15 16:17:04 (9 years ago)

Author:

emanuel.dima@uni-tuebingen.de

Message:

22. alpha 25: removed unused iso-639-2 support and unused german tokenizer; fixed bug related to xml unencoding; changed weblicht config

File:

• SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/Aggregator.java (modified) (9 diffs)

SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/Aggregator.java

@@ -18 +18 @@
 import eu.clarin.sru.fcs.aggregator.rest.RestService;
 import eu.clarin.sru.fcs.aggregator.scan.Statistics;
-import eu.clarin.sru.fcs.aggregator.lang.LanguagesISO693_3;
+import eu.clarin.sru.fcs.aggregator.util.LanguagesISO693;
 import io.dropwizard.Application;
 import io.dropwizard.assets.AssetsBundle;
@@ -25 +25 @@
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -35 +34 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicReference;
-import opennlp.tools.tokenize.TokenizerModel;
 import org.slf4j.LoggerFactory;
 
@@ -88 +86 @@
  * @author edima
  *
+ * TODO: ?use weblicht only to show up in zoomed mode
+ * - send only tcf with only a text layer and language (from the list in params)
+ *
+ * TODO: add the modes described above (except live)
+ *
  * TODO: zoom into the results from a corpus, allow functionality only for
  * the view (search for next set of results)
@@ -95 +98 @@
  * Twan (they did a test, it worked)
  *
+ * TODO: add PiWik support, tracking the following:
+ * - visits, searches, search per corpus
+ *
+ * TODO: BUG: language detection is immediate, in UI; export implications
+ *
  * TODO: websockets
  *
@@ -122 +130 @@
         private AtomicReference<Statistics> searchStatsAtom = new AtomicReference<Statistics>(new Statistics());
 
-        private TokenizerModel tokenizerModel;
         private LanguageDetector languageDetector;
         private TextObjectFactory textObjectFactory;
@@ -235 +242 @@
                 }
 
-                LanguagesISO693_3.getInstance(); // force init
-                initTokenizer();
+                LanguagesISO693.getInstance(); // force init
                 initLanguageDetector();
 
@@ -293 +299 @@
         }
 
-        public TokenizerModel getTokenizerModel() {
-                return tokenizerModel;
-        }
-
         private static void shutdownAndAwaitTermination(AggregatorConfiguration.Params params,
                         ThrottledClient sruClient, ExecutorService scheduler) {
@@ -313 +315 @@
         }
 
-        private void initTokenizer() {
-                TokenizerModel model = null;
-                try {
-                        try (InputStream tokenizerModelDeAsIS = Thread.currentThread().getContextClassLoader().getResourceAsStream(DE_TOK_MODEL)) {
-                                model = new TokenizerModel(tokenizerModelDeAsIS);
-                        }
-                } catch (IOException ex) {
-                        log.error("Failed to load tokenizer model", ex);
-                }
-                tokenizerModel = model;
-        }
-
         public void initLanguageDetector() throws IOException {
                 List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAll();