Changeset 5458 for OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestWorker.java

Timestamp:

07/08/14 11:50:40 (10 years ago)

Author:

Oliver Schonefeld

Message:

• clean up pom.xml
• update some dependencies
• update to Apache Http Client 4.3.x

File:

• OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestWorker.java (modified) (9 diffs)

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestWorker.java

@@ -13 +13 @@
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
-import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.CookieSpecs;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.utils.HttpClientUtils;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.impl.cookie.DateParseException;
-import org.apache.http.impl.cookie.DateUtils;
-import org.apache.http.params.CoreConnectionPNames;
-import org.apache.http.params.CoreProtocolPNames;
-import org.apache.http.params.HttpParams;
-import org.apache.http.util.EntityUtils;
+import org.apache.http.client.utils.DateUtils;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.impl.NoConnectionReuseStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.protocol.BasicHttpContext;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -45 +46 @@
 
 public class HarvestWorker {
+    private static final String USER_AGENT = "OAI-Harvester/1.0.0";
     private static final String OAI_NS = "http://www.openarchives.org/OAI/2.0/";
     private static final String ARG_METADATA_PREFIX = "metadataPrefix";
@@ -68 +70 @@
             LoggerFactory.getLogger(HarvestWorker.class);
     private final AbstractHarvester harvester;
-    private final HttpClient client;
+    private final CloseableHttpClient client;
     private final RequestUriBuilder uriBuilder;
     private final HeaderImpl header;
@@ -77 +79 @@
     HarvestWorker(AbstractHarvester harvester) {
         this.harvester = harvester;
-        // http client
-        client = new DefaultHttpClient();
-        client.getParams().setParameter(CoreProtocolPNames.USER_AGENT,
-                "eu.clarin.cmdi.oai.Harvester/0.0.1");
+
+        // create HTTP client
+        // FIXME: get timeout values from somewhere?
+        final int connectTimeout = 30 * 1000;
+        final int socketTimeout = 180 * 1000;
+        client = createHttpClient(connectTimeout, socketTimeout);
+
         // request stuff
         uriBuilder = new RequestUriBuilder();
@@ -490 +495 @@
         }
         request.addHeader(HEADER_CONNECTION, "close");
-        // FIXME: get timeout values from somewhere?
-        HttpParams params = request.getParams();
-        params.setParameter(CoreConnectionPNames.SO_TIMEOUT, 180000);
-        params.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 30000);
-
-        final HttpResponse response = client.execute(request);
+
         /*
          * FIXME: the following code need serious re-factoring
          */
-        boolean close = false;
+        CloseableHttpResponse response = null;
+        boolean close = true;
         try {
+            response = client.execute(request, new BasicHttpContext());
             job.addToNetworkTime(System.currentTimeMillis() - now);
+
             final int status = response.getStatusLine().getStatusCode();
             if (status == HttpStatus.SC_OK) {
                 delay = 0;
+                close = false;
                 return new Response(job, response, harvester);
             } else {
@@ -511 +515 @@
                     logger.debug("got service unavailable status, retrying " +
                             "after {} seconds", delay);
-                    close = true;
                     return null;
                 } else {
@@ -520 +523 @@
             }
         } catch (IOException e) {
-            close = true;
             throw e;
         } catch (XMLStreamException e) {
-            close = true;
             throw e;
         } catch (HarvesterException e) {
-            close = true;
             throw e;
         } finally {
-            if (close) {
-                /*
-                 * try hard to release HTTP client resources ...
-                 */
-                try {
-                    EntityUtils.consume(response.getEntity());
-                } catch (IOException ex) {
-                    /* IGNORE */
-                }
-
-                /* make sure to release allocated resources */
-                HttpClientUtils.closeQuietly(response);
-
-                if (request != null) {
-                    request.abort();
-                }
+            if (close && (response != null)) {
+                /* make sure. response is closed ... */
+                response.close();
             }
         }
@@ -561 +548 @@
                 delay = Integer.parseInt(v) * 1000;
             } catch (NumberFormatException e) {
-                try {
                     Date date = DateUtils.parseDate(v);
-                    delay = date.getTime() - System.currentTimeMillis();
-                } catch (DateParseException f) {
-                    /* IGNORE */
-                }
+                    if (date != null) {
+                        delay = date.getTime() - System.currentTimeMillis();
+                    }
             }
         }
@@ -615 +600 @@
     }
 
+
+    private static CloseableHttpClient createHttpClient(int connectTimeout,
+            int socketTimeout) {
+        final PoolingHttpClientConnectionManager manager =
+                new PoolingHttpClientConnectionManager();
+        manager.setDefaultMaxPerRoute(8);
+        manager.setMaxTotal(128);
+
+        final SocketConfig socketConfig = SocketConfig.custom()
+                .setSoReuseAddress(true)
+                .setSoLinger(0)
+                .build();
+
+        final RequestConfig requestConfig = RequestConfig.custom()
+                .setAuthenticationEnabled(false)
+                .setRedirectsEnabled(true)
+                .setMaxRedirects(4)
+                .setCircularRedirectsAllowed(false)
+                .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
+                .setConnectTimeout(connectTimeout)
+                .setSocketTimeout(socketTimeout)
+                .setConnectionRequestTimeout(0) /* infinite */
+                .setStaleConnectionCheckEnabled(false)
+                .build();
+
+        return HttpClients.custom()
+                .setUserAgent(USER_AGENT)
+                .setConnectionManager(manager)
+                .setDefaultSocketConfig(socketConfig)
+                .setDefaultRequestConfig(requestConfig)
+                .setConnectionReuseStrategy(new NoConnectionReuseStrategy())
+                .build();
+    }
+
 } // class HarvestWorker