Changeset 1163 for OAIHarvester

Timestamp:

03/11/11 09:52:27 (13 years ago)

Author:

oschonef

Message:

• convert ealiestDatestamp to Date in Identify
• add pluggable parser architecture for additional descriptions in Identify
• add description parsers for OAI-identifier and OAI Dublin Core descriptions

Location:

OAIHarvester/trunk/OAIHarvester/src

Files:

• main/java/eu/clarin/cmdi/oai/harvester/Description.java (added)
• main/java/eu/clarin/cmdi/oai/harvester/Harvester.java (modified) (1 diff)
• main/java/eu/clarin/cmdi/oai/harvester/OAIDublinCoreDescription.java (added)
• main/java/eu/clarin/cmdi/oai/harvester/OAIDublinCoreDescriptionParser.java (added)
• main/java/eu/clarin/cmdi/oai/harvester/OAIIdentifierDescription.java (added)
• main/java/eu/clarin/cmdi/oai/harvester/OAIIdentifierDescriptionParser.java (added)
• main/java/eu/clarin/cmdi/oai/harvester/Repository.java (modified) (3 diffs)
• main/java/eu/clarin/cmdi/oai/harvester/ext (added)
• main/java/eu/clarin/cmdi/oai/harvester/ext/DescriptionParser.java (added)
• main/java/eu/clarin/cmdi/oai/harvester/impl/AbstractHarvester.java (modified) (5 diffs)
• main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestJobImpl.java (modified) (8 diffs)
• main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestWorker.java (modified) (5 diffs)
• main/java/eu/clarin/cmdi/oai/harvester/impl/Response.java (modified) (1 diff)
• test/java/eu/clarin/cmdi/oai/harvester/HarvesterTest.java (modified) (2 diffs)

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/Harvester.java

@@ -3 +3 @@
 import java.net.URI;
 
+import eu.clarin.cmdi.oai.harvester.ext.DescriptionParser;
+
 public interface Harvester {
+
+    public void registerDescriptionParser(DescriptionParser parser)
+            throws HarvesterException;
 
     public HarvestJob createJob(URI repositoryURI, HarvestHandler handler)

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/Repository.java

@@ -1 +1 @@
 package eu.clarin.cmdi.oai.harvester;
+import java.util.Date;
 import java.util.List;
 
@@ -25 +26 @@
     public List<String> getAdminEmail();
 
-    public String getEarliestTimestamp();
+    public Date getEarliestTimestamp();
 
     public DeletedNotion getDeletedNotion();
@@ -33 +34 @@
     public int getCompressionMask();
 
+    public List<Description> getDescriptions();
+
 } // interface Repository

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/AbstractHarvester.java

@@ -4 +4 @@
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
 
 import javax.xml.stream.XMLInputFactory;
@@ -16 +18 @@
 import eu.clarin.cmdi.oai.harvester.Harvester;
 import eu.clarin.cmdi.oai.harvester.HarvesterException;
+import eu.clarin.cmdi.oai.harvester.ext.DescriptionParser;
 
 abstract class AbstractHarvester implements Harvester {
@@ -22 +25 @@
     private final XMLInputFactory factory;
 //    private final XMLValidationSchema schema;
+    private List<DescriptionParser> descriptionParsers =
+        new ArrayList<DescriptionParser>();
 
     protected AbstractHarvester() {
@@ -41 +46 @@
     }
 
+    public final void registerDescriptionParser(DescriptionParser parser)
+            throws HarvesterException {
+        if (parser == null) {
+            throw new NullPointerException("parser == null");
+        }
+        if (findDescriptionParser(parser.getNamespaceURI(),
+                parser.getLocalName()) != null) {
+            throw new HarvesterException("description parser for '{" +
+                    parser.getNamespaceURI() + "}" + parser.getLocalName() +
+                    "' was already registered");
+        }
+        if (descriptionParsers == null) {
+            descriptionParsers = new ArrayList<DescriptionParser>();
+        }
+        descriptionParsers.add(parser);
+    }
+    
     public abstract HarvestJob createJob(URI repositoryURI,
             HarvestHandler handler) throws HarvesterException;
@@ -94 +116 @@
     }
 
+    DescriptionParser findDescriptionParser(String namespaceURI,
+            String localName) {
+        if (descriptionParsers != null) {
+            for (DescriptionParser parser : descriptionParsers) {
+                if (namespaceURI.equals(parser.getNamespaceURI()) &&
+                        localName.equals(parser.getLocalName())) {
+                    return parser;
+                }
+            }
+        }
+        return null;
+    }
+
     abstract void doRunJob(HarvestJobImpl job) throws HarvesterException;
 

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestJobImpl.java

@@ -13 +13 @@
 import javax.xml.stream.XMLStreamReader;
 
+import eu.clarin.cmdi.oai.harvester.Description;
 import eu.clarin.cmdi.oai.harvester.HarvestHandler;
 import eu.clarin.cmdi.oai.harvester.HarvestJob;
@@ -109 +110 @@
     private String protocolVersion;
     private List<String> adminEmail = Collections.emptyList();
-    private String earliestTimestamp;
+    private Date earliestTimestamp;
     private DeletedNotion deletedNotion;
     private Granularity granularity;
     private int compressionMask;
+    private List<Description> descriptions;
     private List<MetadataFormat> metadataFormats;
     private long totalRecordCount = 0;
@@ -277 +279 @@
             return Collections.unmodifiableList(statistics);
         } else {
-            return null;
+            return Collections.emptyList();
         }
     }
@@ -355 +357 @@
     }
 
-    void setEarliestTimestamp(String earliestTimestamp) {
+    void setEarliestTimestamp(Date earliestTimestamp) {
         this.earliestTimestamp = earliestTimestamp;
     }
@@ -377 +379 @@
     int getCompressionMask() {
         return compressionMask;
+    }
+
+    void setDescriptions(List<Description> descriptions) {
+        this.descriptions = descriptions;
     }
 
@@ -539 +545 @@
                 
                 @Override
-                public String getEarliestTimestamp() {
+                public Date getEarliestTimestamp() {
                     return earliestTimestamp;
                 }
@@ -560 +566 @@
                 @Override
                 public List<String> getAdminEmail() {
-                    return adminEmail;
+                    return (adminEmail != null) ?
+                            Collections.unmodifiableList(adminEmail) : null;
+                }
+
+                @Override
+                public List<Description> getDescriptions() {
+                    return (descriptions != null) ?
+                            Collections.unmodifiableList(descriptions) : null;
                 }
             };
@@ -570 +583 @@
         this.metadataFormats = metadataFormats;
         if (handler != null) {
-            handler.onListMetadataFormats(this.metadataFormats);
+            handler.onListMetadataFormats(
+                    Collections.unmodifiableList(this.metadataFormats));
         }
     }

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/HarvestWorker.java

@@ -25 +25 @@
 import org.joda.time.format.DateTimeFormatter;
 
+import eu.clarin.cmdi.oai.harvester.Description;
 import eu.clarin.cmdi.oai.harvester.HarvestJob.State;
 import eu.clarin.cmdi.oai.harvester.HarvesterException;
@@ -31 +32 @@
 import eu.clarin.cmdi.oai.harvester.Repository.DeletedNotion;
 import eu.clarin.cmdi.oai.harvester.Repository.Granularity;
+import eu.clarin.cmdi.oai.harvester.ext.DescriptionParser;
 import eu.clarin.cmdi.oai.harvester.impl.HarvestJobImpl.Task;
 
@@ -187 +189 @@
             job.setAdminEmail(adminEmail);
 
-            s = response.readContent(OAI_NS, "earliestDatestamp", true);
-            job.setEarliestTimestamp(s);
+            /*
+             *  defer setting of earliestDatestamp in job until we know
+             *  the datestamp granularity of the repository 
+             */
+            final String earliestDatestamp =
+                response.readContent(OAI_NS, "earliestDatestamp", true);
 
             s = response.readContent(OAI_NS, "deletedRecord", true);
@@ -203 +209 @@
 
             s = response.readContent(OAI_NS, "granularity", true);
+            Granularity granularity = null;
             if (GRANULARITY_DAYS.equals(s)) {
-                job.setGranularity(Granularity.DAYS);
+                granularity = Granularity.DAYS;
             } else if (GRANULARITY_SECONDS.equals(s)) {
-                job.setGranularity(Granularity.SECONDS);
+                granularity = Granularity.SECONDS;
             } else {
                 throw new HarvesterException(
                         "invalid value for 'granularity': " + s);
             }
+            job.setGranularity(granularity);
+            job.setEarliestTimestamp(parseDate(granularity, earliestDatestamp));
 
             int mask = 0;
@@ -225 +234 @@
             job.setCompressionMask(mask);
 
+            List<Description> descriptions = null;
             while (response.readStart(OAI_NS, "description", false)) {
                 response.consumeWhitespace();
                 final String namespaceURI = response.readNamespaceURI();
-                System.err.println("DESCRIPTION: URI = " + namespaceURI);
+                final String localName = response.peekElementLocalName();
+                DescriptionParser parser = harvester.findDescriptionParser(
+                        namespaceURI, localName);
+                if (parser != null) {
+                    Description desc =
+                        parser.parseDescription(response.getXMLStreamReader());
+                    if (desc != null) {
+                        if (descriptions == null) {
+                            descriptions = new ArrayList<Description>();
+                        }
+                        descriptions.add(desc);
+                    }
+                } else {
+                    System.err.println("skipping DESC " + namespaceURI + ", " +
+                            localName);
+                }
                 response.readEnd(OAI_NS, "description", true);
+                job.setDescriptions(descriptions);
             }
             response.readEnd(OAI_NS, "Identify");

OAIHarvester/trunk/OAIHarvester/src/main/java/eu/clarin/cmdi/oai/harvester/impl/Response.java

@@ -286 +286 @@
         }
         return reader.getNamespaceURI();
+    }
+
+    public String peekElementLocalName() throws XMLStreamException {
+        if (!reader.isStartElement()) {
+            throw new XMLStreamException("not at a start elment event",
+                    reader.getLocation());
+        }
+        return reader.getLocalName();
     }
 

OAIHarvester/trunk/OAIHarvester/src/test/java/eu/clarin/cmdi/oai/harvester/HarvesterTest.java

@@ -89 +89 @@
             }
             System.err.println("        compression: " + sb.toString());
+            final List<Description> descs = repository.getDescriptions();
+            if (descs != null) {
+                for (Description desc : descs) {
+                    System.err.println("        description: " + desc);
+                }
+            }
         }
         
@@ -189 +195 @@
 
         Harvester harvester = SimpleHarvester.newInstance();
-        harvester.setMaxNetworkRetryCount(16);
         try {
+            // configure harvester
+            harvester.setMaxNetworkRetryCount(16);
+            harvester.registerDescriptionParser(
+                    new OAIIdentifierDescriptionParser());
+            harvester.registerDescriptionParser(
+                    new OAIDublinCoreDescriptionParser());
+
             HarvestJob job = harvester.createJob(repos, handler);
 //            Calendar cal = Calendar.getInstance(TimeZone.getDefault());