Changeset 1140 for OAIHarvester

Timestamp:

02/22/11 14:52:02 (13 years ago)

Author:

oschonef

Message:

• make maximum network retry count configurable through Harvester
• re-naming internal methods

Location:

OAIHarvester/trunk/OAIHarvester/src

Files:

• main/java/eu/clarin/cmdi/oai/harvester/Harvester.java (modified) (1 diff)
• main/java/eu/clarin/cmdi/oai/harvester/impl/AbstractHarvester.java (modified) (2 diffs)
• main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestJobImpl.java (modified) (2 diffs)
• main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestWorker.java (modified) (1 diff)
• main/java/eu/clarin/cmdi/oai/harvester/impl/ParallelHarvester.java (modified) (1 diff)
• main/java/eu/clarin/cmdi/oai/harvester/impl/SimpleHarvester.java (modified) (2 diffs)
• test/java/eu/clarin/cmdi/oai/harvester/HarvesterTest.java (modified) (1 diff)

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/Harvester.java

@@ -25 +25 @@
     public void shutdown();
 
+    public int getMaxNetworkRetryCount();
+    
+    public void setMaxNetworkRetryCount(int maxNetworkRetryCount);
+
 } // interface Harvester

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/AbstractHarvester.java

@@ -18 +18 @@
 
 abstract class AbstractHarvester implements Harvester {
+    private final int DEFAULT_MAX_NETWORK_RETRY_COUNT = 5;
+    protected int maxNetworkRetryCount = DEFAULT_MAX_NETWORK_RETRY_COUNT;
     private final XMLInputFactory factory;
 //    private final XMLValidationSchema schema;
@@ -83 +85 @@
     }
 
+    @Override
+    public final int getMaxNetworkRetryCount() {
+        return maxNetworkRetryCount;
+    }
+    
+    @Override
+    public final void setMaxNetworkRetryCount(int maxNetworkRetryCount) {
+        if (maxNetworkRetryCount < 1) {
+            throw new IllegalArgumentException(
+                    "invalid value for maxNetworkRetryCount: " +
+                            maxNetworkRetryCount);
+        }
+        this.maxNetworkRetryCount = maxNetworkRetryCount;
+    }
+
     abstract void doRunJob(HarvestJobImpl job) throws HarvesterException;
 

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestJobImpl.java

@@ -57 +57 @@
     private Task task = Task.IDENTIFY_REPOSITORY;
     private long delayUntil = 0;
-    private int networkTry = 0;
+    private int networkRetryCount = 0;
     private List<String> prefixWorklist = null;
     private String resumptionToken = null;
@@ -348 +348 @@
 
     boolean isHarvestingNewPrefix() {
-        return (resumptionToken == null) && (networkTry == 0);
-    }
-
-    int incNetworkRequestCount() {
-        return ++networkTry;
-    }
-    
-    int getNetworkRequestCount() {
-        return networkTry;
-    }
-
-    void resetNetworkRequestCount() {
+        return (resumptionToken == null) && (networkRetryCount == 0);
+    }
+
+    int incNetworkRetryCount() {
+        return ++networkRetryCount;
+    }
+    
+    int getNetworkRetryCount() {
+        return networkRetryCount;
+    }
+
+    void resetNetworkRetryCount() {
         delayUntil = 0;
-        networkTry = 0;
+        networkRetryCount = 0;
     }
 

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestWorker.java

@@ -122 +122 @@
 
             if (ok) {
-                job.resetNetworkRequestCount();
+                job.resetNetworkRetryCount();
             } else {
                 if (delay > 0) {
                     job.setNetworkRequestDelay(delay);
                 }
-                if (job.incNetworkRequestCount() >= maxNetworkTries) {
+                if (job.incNetworkRetryCount() >= maxNetworkTries) {
                     throw new HarvesterException("too many network retries: " +
-                            job.getNetworkRequestCount());
+                            job.getNetworkRetryCount());
                 }
             }

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/ParallelHarvester.java

@@ -30 +30 @@
                             System.err.println(getId() + " process job " +
                                     job.getId());
-                            worker.process(job, 15);
+                            worker.process(job, maxNetworkRetryCount);
                             if (job.isRunning()) {
                                 if (job.getDelay(TimeUnit.MILLISECONDS) > 0) {

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/SimpleHarvester.java

@@ -47 +47 @@
 
         while (job.isRunning()) {
-            worker.process(job, 15);
+            worker.process(job, maxNetworkRetryCount);
             long delay = -1;
             while (job.isRunning() &&
@@ -53 +53 @@
                 try {
                     System.err.println(job.getId() + ": (" +
-                            job.getNetworkRequestCount() + ") waiting " +
+                            job.getNetworkRetryCount() + ") waiting " +
                             delay + " ...");
                     Thread.sleep(delay);

OAIHarvester/trunk/OAIHarvester/src/test/java/eu/clarin/cmdi/oai/harvester/HarvesterTest.java

@@ -189 +189 @@
 
         Harvester harvester = SimpleHarvester.newInstance();
+        harvester.setMaxNetworkRetryCount(16);
         try {
             HarvestJob job = harvester.createJob(repos, handler);