Changeset 5971 for SRUAggregator
- Timestamp:
- 02/12/15 13:09:57 (9 years ago)
- Location:
- SRUAggregator/trunk
- Files:
-
- 2 added
- 18 edited
- 1 moved
Legend:
- Unmodified
- Added
- Removed
-
SRUAggregator/trunk/aggregator.yml
r5960 r5971 1 1 aggregatorParams: 2 2 CENTER_REGISTRY_URL: http://centres.clarin.eu/restxml/ 3 WEBLICHT_URL: https://weblicht.sfs.uni-tuebingen.de/WebLicht-4/?input= 3 # additionalCQLEndpoints: 4 # - https://clarin.ids-mannheim.de/digibibsru-new 5 # - https://lux17.mpi.nl/ds/cqlsearch 6 # - http://www.meertens.knaw.nl/mimore/srucql/ 7 # - http://gysseling.corpus.taalbanknederlands.inl.nl/gyssru/ 8 # - http://brievenalsbuit.inl.nl/zbsru/ 9 # - http://corpus3.aac.ac.at/ddconsru/ 4 10 11 # AGGREGATOR_FILE_PATH: /data/fcsAggregator/fcsAggregatorCorpora.json 5 12 AGGREGATOR_FILE_PATH: /Users/edima/fcsAggregatorCorpora.json 6 SCAN_MAX_DEPTH: 2 13 14 SCAN_MAX_DEPTH: 1 # recommended 3 7 15 SCAN_TASK_INITIAL_DELAY: 0 8 16 SCAN_TASK_INTERVAL: 12 … … 12 20 SEARCH_MAX_CONCURRENT_REQUESTS_PER_ENDPOINT: 4 13 21 14 15 ENDPOINTS_SCAN_TIMEOUT_MS: 60000 22 ENDPOINTS_SCAN_TIMEOUT_MS: 10000 16 23 ENDPOINTS_SEARCH_TIMEOUT_MS: 30000 17 24 EXECUTOR_SHUTDOWN_TIMEOUT_MS: 1000 25 26 WEBLICHT_URL: https://weblicht.sfs.uni-tuebingen.de/WebLicht-4/?input= 18 27 19 28 # use the simple server factory, run on a single port -
SRUAggregator/trunk/build.sh
r5919 r5971 18 18 cp bower_components/bootstrap/dist/js/bootstrap.min.js $LIBDIR/ 19 19 cp bower_components/jquery/dist/jquery.min.js $LIBDIR/ 20 cp bower_components/jquery/dist/jquery.min.map $LIBDIR/ 20 21 cp bower_components/react/react-with-addons.js $LIBDIR/ 21 22 cp bower_components/react/react-with-addons.min.js $LIBDIR/ -
SRUAggregator/trunk/pom.xml
r5959 r5971 8 8 <groupId>eu.clarin.sru.fcs</groupId> 9 9 <artifactId>Aggregator2</artifactId> 10 <version>2.0.0-alpha-1 6</version>10 <version>2.0.0-alpha-19</version> 11 11 <name>FCS Aggregator</name> 12 12 … … 62 62 <groupId>eu.clarin.sru</groupId> 63 63 <artifactId>sru-client</artifactId> 64 <version>0.9. 4</version>64 <version>0.9.5-DEBUG</version> 65 65 </dependency> 66 66 -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/Aggregator.java
r5960 r5971 88 88 * @author edima 89 89 * 90 * TODO: Send mail to Dieter with centers that do not accept recordSchema 91 * parameter 92 * 93 * TODO: mail to LINDAT-Clarin devel about matching whole word in search 94 * 95 * TODO: routing 96 * 97 * TODO: stats page as tabs 98 * 90 99 * TODO: Download search results as csv, excel, tcf, plain text 91 100 * … … 232 241 ScanCrawlTask task = new ScanCrawlTask(sruScanClient, 233 242 params.CENTER_REGISTRY_URL, params.SCAN_MAX_DEPTH, 234 null, scanCacheAtom, corporaCacheFile, scanStatsAtom, searchStatsAtom); 243 params.additionalCQLEndpoints, 244 null, scanCacheAtom, corporaCacheFile, 245 scanStatsAtom, searchStatsAtom); 235 246 scheduler.scheduleAtFixedRate(task, params.SCAN_TASK_INITIAL_DELAY, 236 247 params.SCAN_TASK_INTERVAL, params.getScanTaskTimeUnit()); -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/app/AggregatorConfiguration.java
r5901 r5971 4 4 import com.fasterxml.jackson.annotation.JsonProperty; 5 5 import io.dropwizard.Configuration; 6 import java.net.URL; 7 import java.util.List; 6 8 import java.util.concurrent.TimeUnit; 7 9 import org.hibernate.validator.constraints.NotEmpty; … … 60 62 long EXECUTOR_SHUTDOWN_TIMEOUT_MS; 61 63 64 @JsonProperty 65 List<URL> additionalCQLEndpoints; 66 62 67 @JsonIgnore 63 68 public TimeUnit getScanTaskTimeUnit() { -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/scan/CenterRegistryForTesting.java
r5957 r5971 18 18 private static final Logger logger = Logger.getLogger(CenterRegistryForTesting.class.getName()); 19 19 private boolean hasChildrenLoaded = false; 20 private List<Institution> centers = new ArrayList<Institution>() { 21 { 22 Institution inst = new Institution("test_IDS", null); 23 inst.addEndpoint("https://clarin.ids-mannheim.de/digibibsru-new"); 24 add(inst); 25 } 26 }; 20 private List<Institution> centers = new ArrayList<Institution>(); 27 21 28 22 @Override -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/scan/Diagnostic.java
r5954 r5971 1 1 package eu.clarin.sru.fcs.aggregator.scan; 2 3 import com.fasterxml.jackson.annotation.JsonProperty; 2 4 3 5 /* … … 6 8 public class Diagnostic { 7 9 8 private String reqEndpointUrl, reqContext;9 private String dgnUri, dgnMessage, dgnDiagnostic;10 @JsonProperty 11 String uri; 10 12 11 public Diagnostic(String reqEndpointUrl, String reqContext, String dgnUri, String dgnMessage, String dgnDiagnostic) { 12 this.reqEndpointUrl = reqEndpointUrl; 13 this.reqContext = reqContext; 14 this.dgnUri = dgnUri; 15 this.dgnMessage = dgnMessage; 16 this.dgnDiagnostic = dgnDiagnostic; 17 } 13 @JsonProperty 14 String message; 18 15 19 public Diagnostic() {20 }16 @JsonProperty 17 String diagnostic; 21 18 22 public String getDgnDiagnostic() { 23 return dgnDiagnostic; 24 } 25 26 public String getDgnMessage() { 27 return dgnMessage; 28 } 29 30 public String getDgnUri() { 31 return dgnUri; 32 } 33 34 public String getReqContext() { 35 return reqContext; 36 } 37 38 public String getReqEndpointUrl() { 39 return reqEndpointUrl; 40 } 41 42 public void setDgnDiagnostic(String dgnDiagnostic) { 43 this.dgnDiagnostic = dgnDiagnostic; 44 } 45 46 public void setDgnMessage(String dgnMessage) { 47 this.dgnMessage = dgnMessage; 48 } 49 50 public void setDgnUri(String dgnUri) { 51 this.dgnUri = dgnUri; 52 } 53 54 public void setReqContext(String reqContext) { 55 this.reqContext = reqContext; 56 } 57 58 public void setReqEndpointUrl(String reqEndpointUrl) { 59 this.reqEndpointUrl = reqEndpointUrl; 19 public Diagnostic(String uri, String message, String diagnostic) { 20 this.uri = uri; 21 this.message = message; 22 this.diagnostic = diagnostic; 60 23 } 61 24 … … 63 26 public int hashCode() { 64 27 https://primes.utm.edu/lists/small/1000.txt 65 return reqEndpointUrl.hashCode() * 967 + reqContext.hashCode() * 797 66 + dgnUri.hashCode() * 1669 + dgnMessage.hashCode() * 31 67 + dgnDiagnostic.hashCode(); 28 return uri.hashCode() * 1669 + message.hashCode() * 31 29 + diagnostic.hashCode(); 68 30 } 69 31 … … 74 36 } 75 37 Diagnostic d = (Diagnostic) obj; 76 return reqEndpointUrl.equals(d.reqEndpointUrl) 77 && reqContext.equals(d.reqContext) 78 && dgnUri.equals(d.dgnUri) 79 && dgnMessage.equals(d.dgnMessage) 80 && dgnDiagnostic.equals(d.dgnDiagnostic); 38 return uri.equals(d.uri) && message.equals(d.message) 39 && diagnostic.equals(d.diagnostic); 81 40 } 82 41 } -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/scan/ScanCrawlTask.java
r5959 r5971 5 5 import java.io.File; 6 6 import java.io.IOException; 7 import java.net.URL; 8 import java.util.List; 9 import java.util.ArrayList; 7 10 import java.util.concurrent.atomic.AtomicReference; 8 11 import org.slf4j.LoggerFactory; … … 24 27 private AtomicReference<Statistics> searchStatisticsAtom; 25 28 private String centerRegistryUrl; 29 private List<URL> additionalCQLEndpoints; 26 30 27 31 public ScanCrawlTask(ThrottledClient sruClient, String centerRegistryUrl, 28 int cacheMaxDepth, EndpointFilter filter, 32 int cacheMaxDepth, List<URL> additionalCQLEndpoints, 33 EndpointFilter filter, 29 34 AtomicReference<Corpora> corporaAtom, File cachedCorpora, 30 35 AtomicReference<Statistics> scanStatisticsAtom, … … 34 39 this.centerRegistryUrl = centerRegistryUrl; 35 40 this.cacheMaxDepth = cacheMaxDepth; 41 this.additionalCQLEndpoints = additionalCQLEndpoints; 36 42 this.filter = filter; 37 43 this.corporaAtom = corporaAtom; … … 47 53 48 54 log.info("ScanCrawlTask: Initiating crawl"); 49 CenterRegistry centerRegistry = new CenterRegistryLive(centerRegistryUrl, filter); 50 // centerRegistry.getCQLInstitutions().add(0, 51 // new Institution("test_IDS", null) { 52 // { 53 // addEndpoint("https://clarin.ids-mannheim.de/digibibsru-new"); 54 // } 55 // }); 56 // CenterRegistry centerRegistry = new CenterRegistryForTesting(); 57 ScanCrawler scanCrawler = new ScanCrawler(centerRegistry, sruClient, cacheMaxDepth); 55 List<Institution> institutions = new ArrayList<Institution>(); 56 if (centerRegistryUrl != null && !centerRegistryUrl.isEmpty()) { 57 institutions = new CenterRegistryLive(centerRegistryUrl, filter).getCQLInstitutions(); 58 } 59 if (additionalCQLEndpoints != null && !additionalCQLEndpoints.isEmpty()) { 60 institutions.add(0, 61 new Institution("ZZZ", null) { 62 { 63 for (URL u : additionalCQLEndpoints) { 64 addEndpoint(u.toExternalForm()); 65 } 66 } 67 }); 68 } 69 ScanCrawler scanCrawler = new ScanCrawler(institutions, sruClient, cacheMaxDepth); 58 70 59 71 log.info("ScanCrawlTask: Starting crawl"); -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/scan/ScanCrawler.java
r5959 r5971 10 10 import eu.clarin.sru.client.SRUScanResponse; 11 11 import eu.clarin.sru.client.SRUTerm; 12 import eu.clarin.sru.client.SRUVersion; 12 13 import eu.clarin.sru.client.fcs.ClarinFCSEndpointDescription; 13 14 import eu.clarin.sru.client.fcs.ClarinFCSEndpointDescription.ResourceInfo; … … 16 17 import eu.clarin.sru.fcs.aggregator.util.Throw; 17 18 import java.net.SocketTimeoutException; 19 import java.util.ArrayList; 18 20 import java.util.HashSet; 19 21 import java.util.List; … … 36 38 private static final org.slf4j.Logger log = LoggerFactory.getLogger(ScanCrawler.class); 37 39 38 private final CenterRegistry centerRegistry;40 private final List<Institution> institutions; 39 41 private final int maxDepth; 40 42 private final CounterLatch latch; … … 42 44 private final Statistics statistics = new Statistics(); 43 45 44 public ScanCrawler( CenterRegistry centerRegistry, ThrottledClient sruClient, int maxDepth) {45 this. centerRegistry = centerRegistry;46 public ScanCrawler(List<Institution> institutions, ThrottledClient sruClient, int maxDepth) { 47 this.institutions = institutions; 46 48 this.sruClient = sruClient; 47 49 this.maxDepth = maxDepth; … … 51 53 public Corpora crawl() { 52 54 Corpora cache = new Corpora(); 53 for (Institution institution : centerRegistry.getCQLInstitutions()) {55 for (Institution institution : institutions) { 54 56 cache.addInstitution(institution); 55 57 Iterable<Endpoint> endpoints = institution.getEndpoints(); … … 68 70 log.info("done crawling"); 69 71 70 for (Institution institution : centerRegistry.getCQLInstitutions()) {72 for (Institution institution : institutions) { 71 73 Iterable<Endpoint> endpoints = institution.getEndpoints(); 72 74 for (Endpoint endp : endpoints) { … … 83 85 final Endpoint endpoint; 84 86 final Corpora corpora; 87 String fullRequestUrl; 85 88 86 89 ExplainTask(final Institution institution, final Endpoint endpoint, final Corpora corpora) { … … 96 99 explainRequest.setExtraRequestData(SRUCQL.EXPLAIN_ASK_FOR_RESOURCES_PARAM, "true"); 97 100 explainRequest.setParseRecordDataEnabled(true); 101 fullRequestUrl = explainRequest.makeURI(SRUVersion.VERSION_1_2).toString(); 98 102 } catch (Throwable ex) { 99 103 log.error("Exception creating explain request for {}: {}", endpoint.getUrl(), ex.getMessage()); … … 113 117 try { 114 118 statistics.addEndpointDatapoint(institution, endpoint, stats.getQueueTime(), stats.getExecutionTime()); 119 120 List<String> rootCollections = new ArrayList<String>(); 115 121 if (response != null && response.hasExtraResponseData()) { 116 122 for (SRUExtraResponseData data : response.getExtraResponseData()) { … … 119 125 statistics.upgradeProtocolVersion(institution, endpoint); 120 126 ClarinFCSEndpointDescription desc = (ClarinFCSEndpointDescription) data; 121 addCorpora(corpora, institution, endpoint, desc.getResources(), null);127 addCorpora(corpora, institution, endpoint, rootCollections, desc.getResources(), null); 122 128 } 123 129 } 124 130 } 131 statistics.addEndpointCollections(institution, endpoint, rootCollections); 125 132 126 133 if (response != null && response.hasDiagnostics()) { 127 134 for (SRUDiagnostic d : response.getDiagnostics()) { 128 135 SRUExplainRequest request = response.getRequest(); 129 Diagnostic diag = new Diagnostic(request.getBaseURI().toString(), null, 130 d.getURI(), d.getMessage(), d.getDetails()); 131 statistics.addEndpointDiagnostic(institution, endpoint, diag); 132 log.info("Diagnostic: {} {}: {} {} {}", diag.getReqEndpointUrl(), diag.getReqContext(), 133 diag.getDgnUri(), diag.getDgnMessage(), diag.getDgnDiagnostic()); 136 Diagnostic diag = new Diagnostic(d.getURI(), d.getMessage(), d.getDetails()); 137 statistics.addEndpointDiagnostic(institution, endpoint, diag, fullRequestUrl); 138 log.info("Diagnostic: {}: {}", fullRequestUrl, diag.message); 134 139 } 135 140 } … … 139 144 log.error("{} Exception in explain callback {}", latch.get(), endpoint.getUrl()); 140 145 log.error("--> ", xc); 146 statistics.addErrorDatapoint(institution, endpoint, xc, fullRequestUrl); 141 147 } finally { 142 148 if (endpoint.getProtocol().equals(FCSProtocolVersion.LEGACY)) { 143 149 new ScanTask(institution, endpoint, null, corpora, 0).start(); 150 Diagnostic diag = new Diagnostic("LEGACY", "Endpoint didn't return any resource on EXPLAIN, presuming legacy support", ""); 151 statistics.addEndpointDiagnostic(institution, endpoint, diag, fullRequestUrl); 144 152 } 145 153 … … 153 161 log.error("{} Error while explaining {}: {}", latch.get(), endpoint.getUrl(), error.getMessage()); 154 162 statistics.addEndpointDatapoint(institution, endpoint, stats.getQueueTime(), stats.getExecutionTime()); 155 statistics.addErrorDatapoint(institution, endpoint, error );163 statistics.addErrorDatapoint(institution, endpoint, error, fullRequestUrl); 156 164 if (Throw.isCausedBy(error, SocketTimeoutException.class)) { 157 165 return; … … 166 174 } 167 175 168 private static void addCorpora(Corpora corpora, Institution institution, Endpoint endpoint, 176 private static void addCorpora(Corpora corpora, 177 Institution institution, Endpoint endpoint, 178 List<String> rootCollections, 169 179 List<ResourceInfo> resources, Corpus parentCorpus) { 170 180 if (resources == null) { … … 180 190 181 191 if (corpora.addCorpus(c, parentCorpus)) { 182 addCorpora(corpora, institution, endpoint, ri.getSubResources(), c); 192 if (rootCollections != null) { 193 rootCollections.add(c.getTitle()); 194 } 195 addCorpora(corpora, institution, endpoint, null, ri.getSubResources(), c); 183 196 } 184 197 } … … 208 221 final Corpora corpora; 209 222 final int depth; 223 String fullRequestUrl; 210 224 211 225 ScanTask(final Institution institution, final Endpoint endpoint, … … 230 244 scanRequest.setExtraRequestData(SRUCQL.SCAN_RESOURCE_INFO_PARAMETER, 231 245 SRUCQL.SCAN_RESOURCE_INFO_PARAMETER_DEFAULT_VALUE); 246 fullRequestUrl = scanRequest.makeURI(SRUVersion.VERSION_1_2).toString(); 232 247 } catch (Throwable ex) { 233 248 log.error("Exception creating scan request for {}: {}", endpoint.getUrl(), ex.getMessage()); … … 255 270 if (corpora.addCorpus(c, parentCorpus)) { 256 271 new ScanTask(institution, endpoint, c, corpora, depth + 1).start(); 272 statistics.addEndpointCollection(institution, endpoint, c.getTitle()); 257 273 } 258 274 } 259 275 } 260 } else if (parentCorpus == null) {261 Corpus c = createCorpus(institution, endpoint, null);262 if (corpora.addCorpus(c, parentCorpus)) {263 new ScanTask(institution, endpoint, c, corpora, depth + 1).start();264 }265 276 } 266 277 267 278 if (response != null && response.hasDiagnostics()) { 268 279 for (SRUDiagnostic d : response.getDiagnostics()) { 269 SRUScanRequest request = response.getRequest(); 270 271 String handle = SRUCQL.SCAN_RESOURCE_PARAMETER + "=" + normalizeHandle(parentCorpus); 272 Diagnostic diag = new Diagnostic(request.getBaseURI().toString(), handle, 273 d.getURI(), d.getMessage(), d.getDetails()); 274 statistics.addEndpointDiagnostic(institution, endpoint, diag); 275 log.info("Diagnostic: {} {}: {} {} {}", diag.getReqEndpointUrl(), diag.getReqContext(), 276 diag.getDgnUri(), diag.getDgnMessage(), diag.getDgnDiagnostic()); 280 Diagnostic diag = new Diagnostic(d.getURI(), d.getMessage(), d.getDetails()); 281 statistics.addEndpointDiagnostic(institution, endpoint, diag, fullRequestUrl); 282 log.info("Diagnostic: {}: {}", fullRequestUrl, diag.message); 277 283 } 278 284 } … … 282 288 log.error("{} Exception in scan callback {}#{}", latch.get(), endpoint.getUrl(), normalizeHandle(parentCorpus)); 283 289 log.error("--> ", xc); 290 statistics.addErrorDatapoint(institution, endpoint, xc, fullRequestUrl); 284 291 } finally { 285 292 latch.decrement(); … … 292 299 log.error("{} Error while scanning {}#{}: {}", latch.get(), endpoint.getUrl(), normalizeHandle(parentCorpus), error.getMessage()); 293 300 statistics.addEndpointDatapoint(institution, endpoint, stats.getQueueTime(), stats.getExecutionTime()); 294 statistics.addErrorDatapoint(institution, endpoint, error );301 statistics.addErrorDatapoint(institution, endpoint, error, fullRequestUrl); 295 302 if (Throw.isCausedBy(error, SocketTimeoutException.class)) { 296 303 return; … … 322 329 private static Corpus createCorpus(Institution institution, Endpoint endpoint, SRUTerm term) { 323 330 Corpus c = new Corpus(institution, endpoint); 324 c.setTitle( term.getDisplayTerm());331 c.setTitle("" + term.getDisplayTerm()); 325 332 String handle = term.getValue(); 326 333 if (handle == null) { -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/scan/Statistics.java
r5959 r5971 2 2 3 3 import com.fasterxml.jackson.annotation.JsonProperty; 4 import eu.clarin.sru.client.SRUClientException;5 4 import java.util.ArrayList; 6 5 import java.util.Collections; … … 17 16 public static class EndpointStats { 18 17 18 private final Object lock = new Object(); 19 19 20 @JsonProperty 20 21 FCSProtocolVersion version = FCSProtocolVersion.LEGACY; 22 23 @JsonProperty 24 List<String> rootCollections = new ArrayList<String>(); 21 25 22 26 List<Long> queueTimes = Collections.synchronizedList(new ArrayList<Long>()); … … 25 29 public static class DiagPair { 26 30 27 public DiagPair(Diagnostic diagnostic, int counter) {31 public DiagPair(Diagnostic diagnostic, String context, int counter) { 28 32 this.diagnostic = diagnostic; 33 this.context = context; 29 34 this.counter = counter; 30 35 } 31 36 37 @JsonProperty 32 38 public Diagnostic diagnostic; 39 @JsonProperty 40 String context; 41 @JsonProperty 33 42 public int counter; 34 43 } … … 36 45 @JsonProperty 37 46 Map<String, DiagPair> diagnostics = Collections.synchronizedMap(new HashMap<String, DiagPair>()); 47 48 public static class ExcPair { 49 50 public ExcPair(JsonException exception, String context, int counter) { 51 this.exception = exception; 52 this.context = context; 53 this.counter = counter; 54 } 55 56 @JsonProperty 57 public JsonException exception; 58 @JsonProperty 59 String context; 60 @JsonProperty 61 public int counter; 62 } 63 38 64 @JsonProperty 39 Map<String, Integer> errors = Collections.synchronizedMap(new HashMap<String, Integer>());65 Map<String, ExcPair> errors = Collections.synchronizedMap(new HashMap<String, ExcPair>()); 40 66 41 67 double avg(List<Long> q) { … … 81 107 }; 82 108 109 private final Object lock = new Object(); 110 83 111 // institution to endpoint to statistics_per_endpoint map 84 112 Map<String, Map<String, EndpointStats>> institutions … … 91 119 public void addEndpointDatapoint(Institution institution, Endpoint endpoint, long enqueuedTime, long executionTime) { 92 120 EndpointStats stats = getEndpointStats(institution, endpoint); 93 stats.queueTimes.add(enqueuedTime); 94 stats.executionTimes.add(executionTime); 95 } 96 97 public void addEndpointDiagnostic(Institution institution, Endpoint endpoint, Diagnostic diag) { 98 EndpointStats stats = getEndpointStats(institution, endpoint); 99 if (!stats.diagnostics.containsKey(diag.getDgnUri())) { 100 stats.diagnostics.put(diag.getDgnUri(), new EndpointStats.DiagPair(diag, 1)); 101 } else { 102 stats.diagnostics.get(diag.getDgnUri()).counter++; 121 synchronized (stats.lock) { 122 stats.queueTimes.add(enqueuedTime); 123 stats.executionTimes.add(executionTime); 103 124 } 104 125 } 105 126 106 public void addE rrorDatapoint(Institution institution, Endpoint endpoint, SRUClientException error) {127 public void addEndpointDiagnostic(Institution institution, Endpoint endpoint, Diagnostic diag, String context) { 107 128 EndpointStats stats = getEndpointStats(institution, endpoint); 108 int number = 0; 109 if (stats.errors.containsKey(error.getMessage())) { 110 number = stats.errors.get(error.getMessage()); 129 synchronized (stats.lock) { 130 if (!stats.diagnostics.containsKey(diag.uri)) { 131 stats.diagnostics.put(diag.uri, new EndpointStats.DiagPair(diag, context, 1)); 132 } else { 133 stats.diagnostics.get(diag.uri).counter++; 134 } 111 135 } 112 stats.errors.put(error.getMessage(), number + 1); 136 } 137 138 public void addErrorDatapoint(Institution institution, Endpoint endpoint, Exception error, String context) { 139 EndpointStats stats = getEndpointStats(institution, endpoint); 140 JsonException jxc = new JsonException(error); 141 synchronized (stats.lock) { 142 if (!stats.errors.containsKey(jxc.message)) { 143 stats.errors.put(jxc.message, new EndpointStats.ExcPair(jxc, context, 1)); 144 } else { 145 stats.errors.get(jxc.message).counter++; 146 } 147 } 113 148 } 114 149 115 150 public void upgradeProtocolVersion(Institution institution, Endpoint endpoint) { 116 151 EndpointStats stats = getEndpointStats(institution, endpoint); 117 stats.version = FCSProtocolVersion.VERSION_1; 152 synchronized (stats.lock) { 153 stats.version = FCSProtocolVersion.VERSION_1; 154 } 155 } 156 157 public void addEndpointCollection(Institution institution, Endpoint endpoint, String collectionName) { 158 EndpointStats stats = getEndpointStats(institution, endpoint); 159 synchronized (stats.lock) { 160 stats.rootCollections.add(collectionName); 161 } 162 } 163 164 public void addEndpointCollections(Institution institution, Endpoint endpoint, List<String> collections) { 165 EndpointStats stats = getEndpointStats(institution, endpoint); 166 synchronized (stats.lock) { 167 stats.rootCollections.addAll(collections); 168 } 118 169 } 119 170 120 171 private EndpointStats getEndpointStats(Institution institution, Endpoint endpoint) { 121 if (!institutions.containsKey(institution.getName())) { 122 institutions.put(institution.getName(), 123 Collections.synchronizedMap(new HashMap<String, EndpointStats>())); 172 EndpointStats stats; 173 synchronized (lock) { 174 if (!institutions.containsKey(institution.getName())) { 175 institutions.put(institution.getName(), 176 Collections.synchronizedMap(new HashMap<String, EndpointStats>())); 177 } 178 Map<String, EndpointStats> esmap = institutions.get(institution.getName()); 179 if (!esmap.containsKey(endpoint.getUrl())) { 180 EndpointStats es = new EndpointStats(); 181 es.version = endpoint.getProtocol(); 182 esmap.put(endpoint.getUrl(), es); 183 } 184 stats = esmap.get(endpoint.getUrl()); 124 185 } 125 Map<String, EndpointStats> esmap = institutions.get(institution.getName()); 126 if (!esmap.containsKey(endpoint.getUrl())) { 127 EndpointStats es = new EndpointStats(); 128 es.version = endpoint.getProtocol(); 129 esmap.put(endpoint.getUrl(), es); 130 } 131 return esmap.get(endpoint.getUrl()); 186 return stats; 132 187 } 133 188 } -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/search/Result.java
r5959 r5971 15 15 import eu.clarin.sru.client.fcs.Resource; 16 16 import eu.clarin.sru.fcs.aggregator.scan.Diagnostic; 17 import eu.clarin.sru.fcs.aggregator.scan.JsonException; 17 18 import java.util.ArrayList; 18 19 import java.util.Collections; … … 35 36 private Request request; 36 37 private List<Kwic> kwics = new ArrayList<Kwic>(); 37 private SRUClientException exception;38 private JsonException exception; 38 39 private List<Diagnostic> diagnostics = new ArrayList<Diagnostic>(); 39 40 … … 45 46 SRUClientException xc) { 46 47 this.request = request; 47 this.exception = xc; 48 if (xc != null) { 49 exception = new JsonException(xc); 50 } 48 51 if (response != null && response.hasRecords()) { 49 52 setResponse(response); … … 75 78 for (SRUDiagnostic d : response.getDiagnostics()) { 76 79 SRUSearchRetrieveRequest srurequest = response.getRequest(); 77 diagnostics.add(new Diagnostic(srurequest.getBaseURI().toString(), 78 srurequest.getQuery(), 79 d.getURI(), d.getMessage(), d.getDetails())); 80 diagnostics.add(new Diagnostic(d.getURI(), d.getMessage(), d.getDetails())); 80 81 } 81 82 } … … 124 125 } 125 126 126 public SRUClientException getException() {127 return exception;127 public List<Diagnostic> getDiagnostics() { 128 return Collections.unmodifiableList(diagnostics); 128 129 } 129 130 130 public List<Diagnostic> getDiagnostics() {131 return Collections.unmodifiableList(diagnostics);131 public JsonException getException() { 132 return exception; 132 133 } 133 134 -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/search/Search.java
r5960 r5971 8 8 import eu.clarin.sru.fcs.aggregator.client.ThrottledClient; 9 9 import eu.clarin.sru.fcs.aggregator.scan.Corpus; 10 import eu.clarin.sru.fcs.aggregator.scan.Diagnostic; 11 import eu.clarin.sru.fcs.aggregator.scan.FCSProtocolVersion; 10 12 import eu.clarin.sru.fcs.aggregator.scan.Statistics; 11 13 import eu.clarin.sru.fcs.aggregator.util.SRUCQL; … … 61 63 searchRequest.setVersion(version); 62 64 searchRequest.setMaximumRecords(maxRecords); 65 FCSProtocolVersion fcsVersion = corpus.getEndpoint().getProtocol(); 63 66 // searchRequest.setRecordSchema( 64 // corpus.getEndpoint().getProtocol().equals(FCSProtocolVersion.LEGACY)67 // fcsVersion.equals(FCSProtocolVersion.LEGACY) 65 68 // ? ClarinFCSRecordData.LEGACY_RECORD_SCHEMA 66 69 // : ClarinFCSRecordData.RECORD_SCHEMA); … … 68 71 searchRequest.setStartRecord(startRecord); 69 72 if (corpus.getHandle() != null) { 70 searchRequest.setExtraRequestData(SRUCQL.SEARCH_CORPUS_HANDLE_PARAMETER, corpus.getHandle()); 73 searchRequest.setExtraRequestData( 74 fcsVersion.equals(FCSProtocolVersion.LEGACY) 75 ? SRUCQL.SEARCH_CORPUS_HANDLE_LEGACY_PARAMETER 76 : SRUCQL.SEARCH_CORPUS_HANDLE_PARAMETER, 77 corpus.getHandle()); 71 78 } 72 79 requests.add(request); 80 81 String url = null; 82 try { 83 url = searchRequest.makeURI(SRUVersion.VERSION_1_2).toString(); 84 } catch (SRUClientException ex) { 85 } 86 final String fullRequestUrl = url; 73 87 74 88 try { … … 81 95 results.add(result); 82 96 requests.remove(request); 83 if (!result.getDiagnostics().isEmpty()) { 84 statistics.addEndpointDiagnostic(corpus.getInstitution(), corpus.getEndpoint(), null); 97 List<Diagnostic> diagnostics = result.getDiagnostics(); 98 if (diagnostics != null && !diagnostics.isEmpty()) { 99 log.error("diagnostic for url: " + response.getRequest().makeURI(SRUVersion.VERSION_1_2)); 100 for (Diagnostic diagnostic : diagnostics) { 101 statistics.addEndpointDiagnostic(corpus.getInstitution(), corpus.getEndpoint(), diagnostic, fullRequestUrl); 102 } 85 103 } 86 104 } catch (Throwable xc) { … … 93 111 try { 94 112 statistics.addEndpointDatapoint(corpus.getInstitution(), corpus.getEndpoint(), stats.getQueueTime(), stats.getExecutionTime()); 95 statistics.addErrorDatapoint(corpus.getInstitution(), corpus.getEndpoint(), xc );113 statistics.addErrorDatapoint(corpus.getInstitution(), corpus.getEndpoint(), xc, fullRequestUrl); 96 114 results.add(new Result(request, null, xc)); 97 115 requests.remove(request); 116 log.error("search.onError: ", xc); 98 117 } catch (Throwable xxc) { 99 118 log.error("search.onError exception:", xxc); -
SRUAggregator/trunk/src/main/java/eu/clarin/sru/fcs/aggregator/util/SRUCQL.java
r5957 r5971 13 13 14 14 public static final String SEARCH_RETRIEVE = "searchRetrieve"; 15 public static final String SEARCH_CORPUS_HANDLE_PARAMETER = "x-cmd-context";16 15 public static final String SEARCH_QUERY_PARAMETER = "query"; 16 public static final String SEARCH_CORPUS_HANDLE_LEGACY_PARAMETER = "x-cmd-context"; 17 public static final String SEARCH_CORPUS_HANDLE_PARAMETER = "x-fcs-context"; 17 18 18 19 public static final String SCAN = "scan"; -
SRUAggregator/trunk/src/main/resources/assets/js/main.js
r5960 r5971 2 2 (function() { 3 3 "use strict"; 4 5 var VERSION = "VERSION 2.0.0.α18"; 4 6 5 7 var PT = React.PropTypes; … … 195 197 }, 196 198 197 renderEndpoint: function(endpoint) { 199 renderCollections: function(colls) { 200 return React.createElement("div", {style: {marginLeft:40}}, 201 colls.length === 0 ? 202 React.createElement("div", {style: {color:"#a94442"}}, "NO collections found") 203 : 204 React.createElement("div", null, 205 colls.length, " root collection(s):", 206 React.createElement("ul", {className: "list-unstyled", style: {marginLeft:40}}, 207 colls.map(function(name) { return React.createElement("div", null, name); }) 208 ) 209 ) 210 211 ); 212 }, 213 214 renderDiagnostic: function(d) { 215 return React.createElement("div", {key: d.diagnostic.uri}, 216 React.createElement("div", {className: "inline alert alert-warning"}, 217 React.createElement("div", null, "Diagnostic: ", d.diagnostic.message, ": ", d.diagnostic.diagnostic), 218 React.createElement("div", null, "Context: ", React.createElement("a", {href: d.context}, d.context)) 219 ), 220 " ", 221 React.createElement("div", {className: "inline"}, React.createElement("span", {className: "badge"}, "x ", d.counter)) 222 ); 223 }, 224 225 renderError: function(e) { 226 var xc = e.exception; 227 return React.createElement("div", {key: xc.message}, 228 React.createElement("div", {className: "inline alert alert-danger", role: "alert"}, 229 React.createElement("div", null, "Exception: ", xc.message), 230 React.createElement("div", null, "Context: ", React.createElement("a", {href: e.context}, e.context)), 231 xc.cause ? React.createElement("div", null, "Caused by: ", xc.cause) : false 232 ), 233 " ", 234 React.createElement("div", {className: "inline"}, React.createElement("span", {className: "badge"}, "x ", e.counter)) 235 ); 236 }, 237 238 renderEndpoint: function(isScan, endpoint) { 198 239 var stat = endpoint[1]; 199 var errors = _. pairs(stat.errors);240 var errors = _.values(stat.errors); 200 241 var diagnostics = _.values(stat.diagnostics); 201 return React.createElement("div", null,202 React.createElement("ul", {className: "list-inline list-unstyled" },242 return React.createElement("div", {style: {marginTop:10}}, 243 React.createElement("ul", {className: "list-inline list-unstyled", style: {marginBottom:0}}, 203 244 React.createElement("li", null, 204 245 stat.version == "LEGACY" ? 205 246 React.createElement("span", {style: {color:'#a94442'}}, "legacy ", React.createElement("i", {className: "glyphicon glyphicon-thumbs-down"}), " ") 206 : false, 247 : React.createElement("span", {style: {color:'#3c763d'}}, React.createElement("i", {className: "glyphicon glyphicon-thumbs-up"}), " "), 248 207 249 " "+endpoint[0], ":" 208 250 ), … … 213 255 ) 214 256 ), 257 isScan ? this.renderCollections(stat.rootCollections) : false, 215 258 (errors && errors.length) ? 216 React.createElement("ul", {className: "list-unstyled inline", style: {marginLeft:40}}, 217 errors.map(function(e) { 218 return React.createElement("div", null, 219 React.createElement("div", {className: "inline alert alert-danger"}, " Exception: ", e[0]), 220 " ", 221 React.createElement("div", {className: "inline"}, React.createElement("span", {className: "badge"}, "x ", e[1])) 222 ); 223 }) 259 React.createElement("div", {className: "inline", style: {marginLeft:40}}, 260 errors.map(this.renderError) 224 261 ) : false, 225 262 226 263 (diagnostics && diagnostics.length) ? 227 React.createElement("ul", {className: "list-unstyled inline", style: {marginLeft:40}}, 228 diagnostics.map(function(d) { 229 return React.createElement("div", null, 230 React.createElement("div", {className: "inline alert alert-warning"}, 231 "Diagnostic: ", d.diagnostic.dgnMessage, ": ", d.diagnostic.dgnDiagnostic 232 ), 233 " ", 234 React.createElement("div", {className: "inline"}, React.createElement("span", {className: "badge"}, "x ", d.counter)) 235 ); 236 }) 264 React.createElement("div", {className: "inline", style: {marginLeft:40}}, 265 diagnostics.map(this.renderDiagnostic) 237 266 ) : false 238 267 … … 240 269 }, 241 270 242 renderInstitution: function(i nst) {271 renderInstitution: function(isScan, inst) { 243 272 return React.createElement("div", {style: {marginBottom:30}}, 244 273 React.createElement("h4", null, inst[0]), 245 React.createElement("div", {style: {marginLeft:20}}, " ", _.pairs(inst[1]).map(this.renderEndpoint ) )274 React.createElement("div", {style: {marginLeft:20}}, " ", _.pairs(inst[1]).map(this.renderEndpoint.bind(this, isScan)) ) 246 275 ); 247 276 }, 248 277 249 renderStatistics: function( stats) {278 renderStatistics: function(isScan, stats) { 250 279 return React.createElement("div", {className: "container"}, 251 280 React.createElement("ul", {className: "list-inline list-unstyled"}, … … 262 291 React.createElement("li", null, "timeout:", " ", React.createElement("kbd", null, stats.timeout, " seconds")) 263 292 ), 264 React.createElement("div", null, " ", _.pairs(stats.institutions).map(this.renderInstitution ), " ")293 React.createElement("div", null, " ", _.pairs(stats.institutions).map(this.renderInstitution.bind(this, isScan)), " ") 265 294 ) 266 295 ; … … 273 302 React.createElement("h1", null, "Statistics"), 274 303 React.createElement("h2", null, "Last scan"), 275 this.renderStatistics(t his.state.lastScanStats),304 this.renderStatistics(true, this.state.lastScanStats), 276 305 React.createElement("h2", null, "Searches since last scan"), 277 this.renderStatistics( this.state.searchStats)306 this.renderStatistics(false, this.state.searchStats) 278 307 ) 279 308 ) … … 414 443 React.createElement("a", {title: "about", href: "#", onClick: this.about}, 415 444 React.createElement("span", {className: "glyphicon glyphicon-info-sign"}), 416 React.createElement("span", null, "VERSION 2.0.0.α16")445 React.createElement("span", null, VERSION) 417 446 ) 418 447 ), -
SRUAggregator/trunk/src/main/resources/assets/js/main.jsx
r5960 r5971 2 2 (function() { 3 3 "use strict"; 4 5 var VERSION = "VERSION 2.0.0.α19"; 4 6 5 7 var PT = React.PropTypes; … … 195 197 }, 196 198 197 renderEndpoint: function(endpoint) { 199 renderCollections: function(colls) { 200 return <div style={{marginLeft:40}}> 201 { colls.length === 0 ? 202 <div style={{color:"#a94442"}}>NO collections found</div> 203 : 204 <div> 205 {colls.length} root collection(s): 206 <ul className='list-unstyled' style={{marginLeft:40}}> 207 { colls.map(function(name) { return <div>{name}</div>; }) } 208 </ul> 209 </div> 210 } 211 </div>; 212 }, 213 214 renderDiagnostic: function(d) { 215 return <div key={d.diagnostic.uri}> 216 <div className="inline alert alert-warning"> 217 <div>Diagnostic: {d.diagnostic.message}: {d.diagnostic.diagnostic}</div> 218 <div>Context: <a href={d.context}>{d.context}</a></div> 219 </div> 220 {" "} 221 <div className="inline"><span className="badge">x {d.counter}</span></div> 222 </div>; 223 }, 224 225 renderError: function(e) { 226 var xc = e.exception; 227 return <div key={xc.message}> 228 <div className="inline alert alert-danger" role="alert"> 229 <div>Exception: {xc.message}</div> 230 <div>Context: <a href={e.context}>{e.context}</a></div> 231 { xc.cause ? <div>Caused by: {xc.cause}</div> : false} 232 </div> 233 {" "} 234 <div className="inline"><span className="badge">x {e.counter}</span></div> 235 </div>; 236 }, 237 238 renderEndpoint: function(isScan, endpoint) { 198 239 var stat = endpoint[1]; 199 var errors = _. pairs(stat.errors);240 var errors = _.values(stat.errors); 200 241 var diagnostics = _.values(stat.diagnostics); 201 return <div >202 <ul className='list-inline list-unstyled' >242 return <div style={{marginTop:10}}> 243 <ul className='list-inline list-unstyled' style={{marginBottom:0}}> 203 244 <li> 204 245 { stat.version == "LEGACY" ? 205 246 <span style={{color:'#a94442'}}>legacy <i className="glyphicon glyphicon-thumbs-down"></i> </span> 206 : false} 247 : <span style={{color:'#3c763d'}}><i className="glyphicon glyphicon-thumbs-up"></i> </span> 248 } 207 249 { " "+endpoint[0] }: 208 250 </li> … … 213 255 </li> 214 256 </ul> 257 { isScan ? this.renderCollections(stat.rootCollections) : false } 215 258 { (errors && errors.length) ? 216 <ul className='list-unstyled inline' style={{marginLeft:40}}> 217 { errors.map(function(e) { 218 return <div > 219 <div className="inline alert alert-danger"> Exception: {e[0]}</div> 220 {" "} 221 <div className="inline"><span className="badge">x {e[1]}</span></div> 222 </div>; 223 }) } 224 </ul> : false 259 <div className='inline' style={{marginLeft:40}}> 260 { errors.map(this.renderError) } 261 </div> : false 225 262 } 226 263 { (diagnostics && diagnostics.length) ? 227 <ul className='list-unstyled inline' style={{marginLeft:40}}> 228 { diagnostics.map(function(d) { 229 return <div > 230 <div className="inline alert alert-warning"> 231 Diagnostic: {d.diagnostic.dgnMessage}: {d.diagnostic.dgnDiagnostic} 232 </div> 233 {" "} 234 <div className="inline"><span className="badge">x {d.counter}</span></div> 235 </div>; 236 }) } 237 </ul> : false 264 <div className='inline' style={{marginLeft:40}}> 265 { diagnostics.map(this.renderDiagnostic) } 266 </div> : false 238 267 } 239 268 </div>; 240 269 }, 241 270 242 renderInstitution: function(i nst) {271 renderInstitution: function(isScan, inst) { 243 272 return <div style={{marginBottom:30}}> 244 273 <h4>{inst[0]}</h4> 245 <div style={{marginLeft:20}}> {_.pairs(inst[1]).map(this.renderEndpoint ) }</div>274 <div style={{marginLeft:20}}> {_.pairs(inst[1]).map(this.renderEndpoint.bind(this, isScan)) }</div> 246 275 </div>; 247 276 }, 248 277 249 renderStatistics: function( stats) {278 renderStatistics: function(isScan, stats) { 250 279 return <div className="container"> 251 280 <ul className='list-inline list-unstyled'> … … 262 291 <li>timeout:{" "}<kbd>{stats.timeout} seconds</kbd></li> 263 292 </ul> 264 <div> { _.pairs(stats.institutions).map(this.renderInstitution ) } </div>293 <div> { _.pairs(stats.institutions).map(this.renderInstitution.bind(this, isScan)) } </div> 265 294 </div> 266 295 ; … … 273 302 <h1>Statistics</h1> 274 303 <h2>Last scan</h2> 275 {this.renderStatistics(t his.state.lastScanStats)}304 {this.renderStatistics(true, this.state.lastScanStats)} 276 305 <h2>Searches since last scan</h2> 277 {this.renderStatistics( this.state.searchStats)}306 {this.renderStatistics(false, this.state.searchStats)} 278 307 </div> 279 308 </div> … … 414 443 <a title="about" href="#" onClick={this.about}> 415 444 <span className="glyphicon glyphicon-info-sign"></span> 416 <span> VERSION 2.0.0.α16</span>445 <span>{VERSION}</span> 417 446 </a> 418 447 </div> -
SRUAggregator/trunk/src/main/resources/assets/js/search.js
r5960 r5971 219 219 return; 220 220 } 221 var selectedIds = this.state.corpora.getSelectedIds(); 222 // console.log("searching in the following corpora:", selectedIds); 221 223 this.props.ajax({ 222 224 url: 'rest/search', … … 227 229 query: query, 228 230 numberOfResults: this.state.numberOfResults, 229 corporaIds: this.state.corpora.getSelectedIds(),231 corporaIds: selectedIds, 230 232 }, 231 233 success: function(searchId, textStatus, jqXHR) { … … 595 597 }, 596 598 599 renderDiagnostic: function(d) { 600 return React.createElement("div", {className: "alert alert-warning"}, 601 React.createElement("div", null, "Diagnostic: ", d.diagnostic.message) 602 ); 603 }, 604 597 605 renderDiagnostics: function(corpusHit) { 598 606 if (!corpusHit.diagnostics || corpusHit.diagnostics.length === 0) { … … 600 608 } 601 609 602 return corpusHit.diagnostics.map(function(d) { 603 return React.createElement("div", {className: "alert alert-danger", role: "alert", key: d.dgnUri}, 604 d.dgnMessage, ": ", d.dgnDiagnostic 605 ); 606 }); 610 return corpusHit.diagnostics.map(this.renderDiagnostic); 611 }, 612 613 renderErrors: function(corpusHit) { 614 var xc = corpusHit.exception; 615 if (!xc) { 616 return false; 617 } 618 return ( 619 React.createElement("div", {className: "alert alert-danger", role: "alert"}, 620 React.createElement("div", null, "Exception: ", xc.message), 621 xc.cause ? React.createElement("div", null, "Caused by: ", xc.cause) : false 622 ) 623 ); 607 624 }, 608 625 … … 611 628 if (this.state.displayKwic) { 612 629 return React.createElement("div", null, 630 this.renderErrors(corpusHit), 613 631 this.renderDiagnostics(corpusHit), 614 632 React.createElement("table", {className: "table table-condensed table-hover", style: fulllength}, … … 618 636 } else { 619 637 return React.createElement("div", null, 638 this.renderErrors(corpusHit), 620 639 this.renderDiagnostics(corpusHit), 621 640 corpusHit.kwics.map(this.renderRowsAsHits) … … 625 644 626 645 renderResultPanels: function(corpusHit) { 627 if (corpusHit.kwics.length === 0 && 646 if (corpusHit.kwics.length === 0 && 647 !corpusHit.exception && 628 648 corpusHit.diagnostics.length === 0) { 629 return false;649 return false; 630 650 } 631 651 return React.createElement(Panel, {key: corpusHit.corpus.title, … … 637 657 638 658 renderToolbox: function() { 659 if (this.props.requests.length > 0) { 660 return false; 661 } 639 662 return React.createElement("div", {className: "toolbox float-left"}, 640 663 React.createElement("a", {className: "btn btn-default", href: this.props.getDownloadLink("text")}, -
SRUAggregator/trunk/src/main/resources/assets/js/search.jsx
r5960 r5971 219 219 return; 220 220 } 221 var selectedIds = this.state.corpora.getSelectedIds(); 222 // console.log("searching in the following corpora:", selectedIds); 221 223 this.props.ajax({ 222 224 url: 'rest/search', … … 227 229 query: query, 228 230 numberOfResults: this.state.numberOfResults, 229 corporaIds: this.state.corpora.getSelectedIds(),231 corporaIds: selectedIds, 230 232 }, 231 233 success: function(searchId, textStatus, jqXHR) { … … 595 597 }, 596 598 599 renderDiagnostic: function(d) { 600 return <div className="alert alert-warning"> 601 <div>Diagnostic: {d.diagnostic.message}</div> 602 </div>; 603 }, 604 597 605 renderDiagnostics: function(corpusHit) { 598 606 if (!corpusHit.diagnostics || corpusHit.diagnostics.length === 0) { … … 600 608 } 601 609 602 return corpusHit.diagnostics.map(function(d) { 603 return <div className="alert alert-danger" role="alert" key={d.dgnUri}> 604 {d.dgnMessage}{": "}{d.dgnDiagnostic} 605 </div>; 606 }); 610 return corpusHit.diagnostics.map(this.renderDiagnostic); 611 }, 612 613 renderErrors: function(corpusHit) { 614 var xc = corpusHit.exception; 615 if (!xc) { 616 return false; 617 } 618 return ( 619 <div className="alert alert-danger" role="alert"> 620 <div>Exception: {xc.message}</div> 621 { xc.cause ? <div>Caused by: {xc.cause}</div> : false} 622 </div> 623 ); 607 624 }, 608 625 … … 611 628 if (this.state.displayKwic) { 612 629 return <div> 630 {this.renderErrors(corpusHit)} 613 631 {this.renderDiagnostics(corpusHit)} 614 632 <table className="table table-condensed table-hover" style={fulllength}> … … 618 636 } else { 619 637 return <div> 638 {this.renderErrors(corpusHit)} 620 639 {this.renderDiagnostics(corpusHit)} 621 640 {corpusHit.kwics.map(this.renderRowsAsHits)} … … 625 644 626 645 renderResultPanels: function(corpusHit) { 627 if (corpusHit.kwics.length === 0 && 646 if (corpusHit.kwics.length === 0 && 647 !corpusHit.exception && 628 648 corpusHit.diagnostics.length === 0) { 629 return false;649 return false; 630 650 } 631 651 return <Panel key={corpusHit.corpus.title} … … 637 657 638 658 renderToolbox: function() { 659 if (this.props.requests.length > 0) { 660 return false; 661 } 639 662 return <div className="toolbox float-left"> 640 663 <a className="btn btn-default" href={this.props.getDownloadLink("text")}> -
SRUAggregator/trunk/src/test/java/eu/clarin/sru/fcs/aggregator/app/CQLEnumerationTest.java
r5957 r5971 5 5 import eu.clarin.sru.fcs.aggregator.scan.Endpoint; 6 6 import eu.clarin.sru.fcs.aggregator.scan.Institution; 7 import java.util.HashSet; 8 import java.util.List; 9 import java.util.Set; 7 10 import javax.naming.NamingException; 8 11 import org.junit.Ignore; … … 17 20 public class CQLEnumerationTest { 18 21 19 @Test 20 public void test() throws NamingException { 22 public void printAll(String centerRegistryUrl) throws NamingException { 21 23 22 24 try { 23 String centerRegistryUrl = "https://centres-staging.clarin.eu:4430/restxml/";24 25 CenterRegistry centerRegistry = new CenterRegistryLive(centerRegistryUrl, null); 25 for (Institution institution : centerRegistry.getCQLInstitutions()) { 26 System.out.println(institution.getName() + ": "); 26 List<Institution> list = centerRegistry.getCQLInstitutions(); 27 for (Institution institution : list) { 28 System.out.println("1: " + institution.getName() + ": "); 27 29 for (Endpoint e : institution.getEndpoints()) { 28 System.out.println(" \t -> " + e);30 System.out.println("1: \t -> " + e); 29 31 } 30 32 } 31 Assert.assertTrue(centerRegistry.getCQLInstitutions().size() > 10); 33 Assert.assertTrue(list.size() > 10); 34 } finally { 35 } 36 } 37 38 @Test 39 public void testPrintAll() throws NamingException { 40 System.out.println("Official registry:\n========================"); 41 printAll("http://centerregistry-clarin.esc.rzg.mpg.de/restxml/"); 42 43 System.out.println("Testing registry:\n========================"); 44 printAll("https://centres-staging.clarin.eu:4430/restxml/"); 45 } 46 47 @Test 48 public void testEq() throws NamingException { 49 50 try { 51 Set<Endpoint> list1, list2; 52 { 53 String centerRegistryUrl = "http://centerregistry-clarin.esc.rzg.mpg.de/restxml/"; 54 CenterRegistry centerRegistry = new CenterRegistryLive(centerRegistryUrl, null); 55 list1 = new HashSet<Endpoint>(); 56 for (Institution i : centerRegistry.getCQLInstitutions()) { 57 list1.addAll(i.getEndpoints()); 58 } 59 } 60 61 { 62 String centerRegistryUrl = "https://centres-staging.clarin.eu:4430/restxml/"; 63 CenterRegistry centerRegistry = new CenterRegistryLive(centerRegistryUrl, null); 64 list2 = new HashSet<Endpoint>(); 65 for (Institution i : centerRegistry.getCQLInstitutions()) { 66 list2.addAll(i.getEndpoints()); 67 } 68 } 69 70 Assert.assertTrue(list1.size() > 10); 71 Assert.assertTrue(list2.size() > 10); 72 73 for (Endpoint e : list1) { 74 Assert.assertTrue("testing registry does not contain " + e, list2.contains(e)); 75 } 76 for (Endpoint e : list2) { 77 Assert.assertTrue("official registry does not contain " + e, list1.contains(e)); 78 } 32 79 } finally { 33 80 } -
SRUAggregator/trunk/src/test/java/eu/clarin/sru/fcs/aggregator/app/ScanCrawlerTest.java
r5957 r5971 37 37 InitialContext context = new InitialContext(); 38 38 String centerRegistryUrl = (String) context.lookup("java:comp/env/center-registry-url"); 39 CenterRegistry centerRegistry = new CenterRegistryLive(centerRegistryUrl, filter); 40 ScanCrawler crawler = new ScanCrawler(centerRegistry, sruClient, 2); 39 ScanCrawler crawler = new ScanCrawler( 40 new CenterRegistryLive(centerRegistryUrl, filter).getCQLInstitutions(), 41 sruClient, 2); 41 42 Corpora cache = crawler.crawl(); 42 43 Corpus tueRootCorpus = cache.findByEndpoint("http://weblicht.sfs.uni-tuebingen.de/rws/sru/").get(0);
Note: See TracChangeset
for help on using the changeset viewer.