Changeset 1144 for OAIHarvester

Timestamp:

02/24/11 13:57:42 (13 years ago)

Author:

oschonef

Message:

• add per prefix statistics

Location:

OAIHarvester/trunk/OAIHarvester/src

Files:

• main/java/eu/clarin/cmdi/oai/harvester/HarvestJob.java (modified) (3 diffs)
• main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestJobImpl.java (modified) (11 diffs)
• main/java/eu/clarin/cmdi/oai/harvester/impl/HeaderImpl.java (modified) (2 diffs)
• test/java/eu/clarin/cmdi/oai/harvester/HarvesterTest.java (modified) (4 diffs)

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/HarvestJob.java

@@ -11 +11 @@
         ERROR,
         CANCELED
-    };
+    } // enum State
+
+    public interface Statistics {
+        public String getPrefix();
+        
+        public long getRecordCount();
+        
+        public long getRequestCount();
+        
+        public long getResumptionCount();
+
+        public long getBytesTransferred();
+        
+        public Date getLatestDatestamp();
+    } // interface Statistics
 
     public long getId();
@@ -19 +33 @@
     public boolean isRunning();
 
-    public long getRecordCount();
+    public long getTotelRecordCount();
 
-    public long getRequestCount();
+    public long getTotelRequestCount();
 
-    public long getResumptionCount();
-
-    public long getBytesTransferred();
+    public long getTotalBytesTransferred();
 
     public long getTotalTime();
@@ -53 +65 @@
     public void setMetadataPrefixes(List<String> prefixes);
 
+    public Statistics getStatistics(String prefix);
+    
+    public List<Statistics> getStatistics();
+
     public void run() throws HarvesterException;
 

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestJobImpl.java

@@ -28 +28 @@
         HARVEST_RECORDS
     }
+    private static final class StatisticsImpl implements Statistics {
+        private final String prefix;
+        private long recordCount = 0;
+        private long requestCount = 0;
+        private long resumptionCount = 0;
+        private long bytedTransferred = 0;
+        private Date latestDatestamp = null;
+        
+        private StatisticsImpl(String prefix) {
+            this.prefix = prefix;
+        }
+
+        @Override
+        public String getPrefix() {
+            return prefix;
+        }
+
+        @Override
+        public long getRecordCount() {
+            return recordCount;
+        }
+
+        private void incRecordCount() {
+            this.recordCount++;
+        }
+
+        @Override
+        public long getRequestCount() {
+            return requestCount;
+        }
+
+        private void incRequestCount() {
+            this.requestCount++;
+        }
+
+        @Override
+        public long getResumptionCount() {
+            return resumptionCount;
+        }
+
+        private void incResumptionCount() {
+            this.resumptionCount++;
+        }
+
+        @Override
+        public long getBytesTransferred() {
+            return bytedTransferred;
+        }
+    
+        private void incBytesTransferred(long bytes) {
+            this.bytedTransferred += bytes;
+        }
+
+        @Override
+        public Date getLatestDatestamp() {
+            return latestDatestamp;
+        }
+        
+        private void updateLatestDatestmp(Date latestDatestamp) {
+            if (this.latestDatestamp == null) {
+                this.latestDatestamp = latestDatestamp;
+            } else {
+                if (latestDatestamp.compareTo(this.latestDatestamp) > 0) {
+                    this.latestDatestamp = latestDatestamp;
+                }
+            }
+        }
+    } // inner class Statistics
     private final AbstractHarvester harvester;
     private final long id;
@@ -46 +114 @@
     private int compressionMask;
     private List<MetadataFormat> metadataFormats;
-    private long recordCount = 0;
-    private long requestCount = 0;
-    private long resumptionCount = 0;
-    private long bytedTransferred = 0;
+    private long totalRecordCount = 0;
+    private long totalRequestCount = 0;
+    private long totalBytesTransferred = 0;
     private long timeTotal = 0;
     private long timeNetwork = 0;
@@ -55 +122 @@
     private long timeProcessing = 0;
     private long startTimestamp;
+    private List<Statistics> statistics = null;
     private Task task = Task.IDENTIFY_REPOSITORY;
     private long delayUntil = 0;
@@ -60 +128 @@
     private List<String> prefixWorklist = null;
     private String resumptionToken = null;
-    private Date latestDatestamp = null;
 
     HarvestJobImpl(AbstractHarvester harvester, long id, URI repositoryURI,
@@ -86 +153 @@
 
     @Override
-    public long getRecordCount() {
-        return recordCount;
-    }
-
-    @Override
-    public long getRequestCount() {
-        return requestCount;
-    }
-
-    @Override
-    public long getResumptionCount() {
-        return resumptionCount;
-    }
-
-    @Override
-    public long getBytesTransferred() {
-        return bytedTransferred;
+    public long getTotelRecordCount() {
+        return totalRecordCount;
+    }
+
+    @Override
+    public long getTotelRequestCount() {
+        return totalRequestCount;
+    }
+
+    @Override
+    public long getTotalBytesTransferred() {
+        return totalBytesTransferred;
     }
 
@@ -188 +250 @@
         } else {
             this.metadataPrefixes = null;
+        }
+    }
+    
+    @Override
+    public Statistics getStatistics(String prefix) {
+        if (prefix == null) {
+            throw new NullPointerException("prefix == null");
+        }
+        if ((state == State.NEW) || (state == State.RUNNING)) {
+            throw new IllegalStateException("invalid state");
+        }
+        for (Statistics stats : statistics) {
+            if (prefix.equals(stats.getPrefix())) {
+                return stats;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public List<Statistics> getStatistics() {
+        if ((state == State.NEW) || (state == State.RUNNING)) {
+            throw new IllegalStateException("invalid state");
+        }
+        if ((statistics != null) && !statistics.isEmpty()) {
+            return Collections.unmodifiableList(statistics);
+        } else {
+            return null;
         }
     }
@@ -290 +380 @@
 
     void incRequestCount() {
-        requestCount++;
+        totalRequestCount++;
+        final StatisticsImpl stats = getCurrentStatistics();
+        if (stats != null) {
+            stats.incRequestCount();
+        }
     }
 
     void incResumptionCount() {
-        resumptionCount++;
+        final StatisticsImpl stats = getCurrentStatistics();
+        if (stats != null) {
+            stats.incResumptionCount();
+        }
     }
 
@@ -303 +400 @@
     void finishRequest(long bytesTransferred, long timeProcessingDelta) {
         timeProcessing += timeProcessingDelta;
-        bytedTransferred += bytesTransferred;
+        totalBytesTransferred += bytesTransferred;
+        final StatisticsImpl stats = getCurrentStatistics();
+        if (stats != null) {
+            stats.incBytesTransferred(bytesTransferred);
+        }
     }
 
@@ -370 +471 @@
 
     void onStartListingRecords() {
-        if (handler != null) {
-            handler.onStartListingRecords(getCurrentPrefix());
+        final String prefix = getCurrentPrefix();
+        if (statistics == null) {
+            statistics = new ArrayList<Statistics>();
+        }
+        statistics.add(0, new StatisticsImpl(prefix));
+
+        if (handler != null) {
+            handler.onStartListingRecords(prefix);
         }
     }
 
     void onFinishListingRecords() {
+        Date latestDatestamp = null;
+
+        final StatisticsImpl stats = getCurrentStatistics();
+        if (stats != null) {
+            latestDatestamp = stats.getLatestDatestamp();
+        }
         if (handler != null) {
             handler.onFinishListingRecords(getCurrentPrefix(), latestDatestamp);
@@ -381 +494 @@
         // reset ...
         resumptionToken = null;
-        latestDatestamp = null;
     }
 
     void onRecord(Header header) {
-        recordCount++;
-        if (latestDatestamp == null) {
-            latestDatestamp = header.getDatestamp();
-        } else {
-            if (latestDatestamp.compareTo(header.getDatestamp()) < 0) {
-                latestDatestamp = header.getDatestamp();
-            }
+        totalRecordCount++;
+        StatisticsImpl stats = getCurrentStatistics();
+        if (stats != null) {
+            stats.incRecordCount();
+            stats.updateLatestDatestmp(header.getDatestamp());
         }
         if (handler != null) {
@@ -472 +582 @@
     }
 
+    private StatisticsImpl getCurrentStatistics() {
+        if ((statistics != null) && !statistics.isEmpty()) {
+            return (StatisticsImpl) statistics.get(0);
+        }
+        return null;
+    }
+
 } // class HarvestJobImpl

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/HeaderImpl.java

@@ -2 +2 @@
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.List;
@@ -44 +45 @@
     @Override
     public List<String> getSets() {
-        return sets.isEmpty() ? null : sets;
+        return sets.isEmpty() ? null : Collections.unmodifiableList(sets);
     }
 

OAIHarvester/trunk/OAIHarvester/src/test/java/eu/clarin/cmdi/oai/harvester/HarvesterTest.java

@@ -6 +6 @@
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
@@ -196 +195 @@
 //            cal.set(2010, Calendar.DECEMBER, 15);
 //            job.setFrom(cal.getTime());
-            job.setMetadataPrefixes(Arrays.asList("oai_dc"));
+//            job.setMetadataPrefixes(Arrays.asList("oai_dc", "cmdi"));
             job.run();
 
@@ -207 +206 @@
 
             System.err.println("==> " + job.getState() + ": "+
-                    job.getRecordCount() + " record(s) [resumed: " +
-                    job.getResumptionCount() + " time(s)] / " +
-                    job.getRequestCount() + " request(s) / " +
-                    job.getBytesTransferred() + " bytes transferred");
+                    job.getTotelRecordCount() + " total record(s) / " +
+                    job.getTotelRequestCount() + " total request(s) / " +
+                    job.getTotalBytesTransferred() +
+                        " total bytes transferred");
             System.err.println("          " +
                     (job.getTotalTime() / 1000f) + " total / " +
@@ -216 +215 @@
                     (job.getWaitTime() / 1000f) + " wait / " +
                     (job.getProcessingTime() / 1000f) + " processing");
+            for (HarvestJob.Statistics stats : job.getStatistics()) {
+                System.err.println("          " + stats.getPrefix() + ": " +
+                        stats.getRecordCount() + " record(s) / " +
+                        stats.getRequestCount() + " requsts(s) [" +
+                        stats.getResumptionCount()  + " time(s) resumed] / " +
+                        "latestDatestamp = " + stats.getLatestDatestamp());
+            }
         } catch (HarvesterProtocolErrorException e) {
             System.err.println("Protocol error: ");