Changeset 3012 for OAIHarvester

Timestamp:

06/10/13 22:02:39 (11 years ago)

Author:

oschonef

Message:

• fix storing raw responses in ZIP file with very large results

File:

• OAIHarvester/trunk/OAIHarvester/src/test/java/eu/clarin/cmdi/oai/harvester/HarvesterTest.java (modified) (4 diffs)

OAIHarvester/trunk/OAIHarvester/src/test/java/eu/clarin/cmdi/oai/harvester/HarvesterTest.java

@@ -1 +1 @@
 package eu.clarin.cmdi.oai.harvester;
 
-import java.io.BufferedInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.Date;
 import java.util.List;
 import java.util.Properties;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipException;
@@ -38 +42 @@
         private int reqNum;
         private String prefix;
-        private byte[] buffer = new byte[8192];
+        private byte[] buffer = new byte[16 * 1024];
 
         public MyHarvestHandler(File file, boolean saveHeader) {
@@ -55 +59 @@
                 if (output != null) {
                     output.flush();
+                    output.finish();
                     output.close();
                     output = null;
@@ -194 +199 @@
         @Override
         public InputStream wrap(InputStream stream) throws IOException {
-            final int size = 16*1024*1024;
-            BufferedInputStream in =
-                new BufferedInputStream(stream, size);
-            in.mark(size);
-            String x = Integer.toHexString(reqNum++);
-            while (x.length() < 4) {
-                x = "0" + x;
-            }
-            final String filename = "request/0x" + x + ".xml";
-            System.err.println("FN: " + filename);
-            ZipEntry entry = new ZipEntry(filename);
-            output.putNextEntry(entry);
-            int r = -1;
-            do {
-                r = in.read(buffer, 0, buffer.length - 1);
-                if (r > 0) {
-                    output.write(buffer, 0, r);
-                }
-            } while (r > 0);
-            output.closeEntry();
-            output.flush();
-            in.reset();
-            return in;
+            /*
+             * !!! WARNING WARNING WARNING WARNING WARNING WARNING WARNING !!!
+             * Do not use this in production. This is just for demo purposes
+             * to also store the raw requests in the ZIP file. This code is
+             * highly inefficient!
+             * For production, just "return stream" in this method. If you're
+             * using the HarvestHandlerAdapter class as super class, just do
+             * not override this method.
+             * !!! WARNING WARNING WARNING WARNING WARNING WARNING WARNING !!!
+             *
+             */
+            return new FilterInputStream(stream) {
+                private final File tmpFile =
+                        File.createTempFile("oai-harvester", null);
+                private final OutputStream tmpStream =
+                        new GZIPOutputStream(new FileOutputStream(tmpFile));
+
+                @Override
+                public int read(byte[] b, int off, int len) throws IOException {
+                    final int read = super.read(b, off, len);
+                    if (read > 0) {
+                        tmpStream.write(b, off, read);
+                    }
+                    return read;
+                }
+
+                @Override
+                public void close() throws IOException {
+                    super.close();
+
+                    try {
+                        tmpStream.close();
+
+                        final String filename =
+                                String.format("request/0x%04X.xml", reqNum++);
+                        System.err.println("FN: " + filename);
+
+                        InputStream in = null;
+                        try {
+                            ZipEntry entry = new ZipEntry(filename);
+                            output.putNextEntry(entry);
+
+                            in = new GZIPInputStream(
+                                    new FileInputStream(tmpFile));
+                            for (;;) {
+                                int read = in.read(buffer);
+                                if (read <= 0) {
+                                    break;
+                                }
+                                output.write(buffer, 0, read);
+                            }
+                            in.close();
+                        } catch (IOException e) {
+                            e.printStackTrace();
+                        } finally {
+                            if (in != null) {
+                                try {
+                                    in.close();
+                                } catch (IOException e) {
+                                }
+                            }
+                            output.closeEntry();
+                        }
+                    } finally {
+                        tmpFile.delete();
+                    }
+                }
+            };
         }