Index: /metadata/trunk/toolkit/scripts/lrt2cmdi.2.py
===================================================================
--- /metadata/trunk/toolkit/scripts/lrt2cmdi.2.py	(revision 2087)
+++ /metadata/trunk/toolkit/scripts/lrt2cmdi.2.py	(revision 2087)
@@ -0,0 +1,327 @@
+#!/usr/bin/env python
+
+# converts the CSV from the LRT inventory to nice and clean CMDI
+# Dieter says: I deny the existance of this script!
+
+import urllib, csv, datetime, pdb, xml.etree.ElementTree as ElementTree
+from curses.ascii import ascii
+
+class CmdiFile:
+    def __init__(self, nodeId) :
+        template            = open("cmdi-lrt-template.xml").read()
+        self.nodeId         = nodeId
+        self.xmlTree        = ElementTree.ElementTree(ElementTree.fromstring(template))
+        self.parentmap      = dict((c, p) for p in self.xmlTree.getiterator() for c in p)
+        self.fillElement("//MdCreationDate", datetime.datetime.now().strftime("%Y-%m-%d"))
+        self.fillElement("//MdSelfLink", "http://www.clarin.eu/node/%s" % nodeId)
+
+    def fillElement(self, xpath, value):
+        self.xmlTree.find(xpath).text = value.strip()
+
+    def fillOptionalElement(self, XPath, value):
+
+        try :
+            result = self.fillElement(XPath, value)
+        except :
+            print "Error in filling optional element"
+
+            pdb.set_trace()
+        else :
+            return result
+
+        ### Conceptual code that should remove optional element that are filled with an empty text.
+        # optional_element_parent_XPath   = XPath + "/.." 
+        # optional_element_parent         = self.xmlTree.find(optional_element_parent_XPath)
+        # optional_element                = self.xmlTree.find(XPath)
+
+        # try :
+        #     assert(optional_element_parent is not None)
+        #     assert(optional_element is not None)
+        # except :
+        #     import pdb
+        #     pdb.set_trace()
+
+        # value = str(value).strip()
+        # if len(value) > 1 :
+        #     optional_element.text   = value
+        # else :
+        #     optional_element_parent.remove(optional_element)
+
+    def fillMultipleElement(self, elementname, xpath, values):
+        # fill in the already existing element
+        if (values[0]):
+            self.fillElement(xpath, values[0])
+
+        element = self.xmlTree.find(xpath)
+        parent = self.parentmap[element]
+        position = parent.getchildren().index(element)
+
+        # then add siblings for the other elements
+        for value in values[1:]:
+            if value:
+                # create new sibling of xpath (elementname) = value
+                position += 1
+                newElement = ElementTree.Element(elementname)
+                newElement.text = value.strip()
+                parent.insert(position, newElement)
+
+    def removeEmptyNodes(self):
+        # we maybe added some elements so need to recalculate the parentmap
+        self.parentmap = dict((c, p) for p in self.xmlTree.getiterator() for c in p)
+
+        removeList = ["ResourceType", "BeginYearResourceCreation", "FinalizationYearResourceCreation", "Institute", \
+                      "DistributionType", "NonCommercialUsageOnly", "UsageReportRequired", "ModificationsRequireRedeposition", "WorkingLanguages", "Date"]
+        for r in removeList:
+            results = self.xmlTree.findall("//%s" % r)
+            for res in results:
+                if not res.text:
+                    parentNode = self.parentmap[res]
+                    parentNode.remove(res)
+
+    def serialize(self):
+        self.removeEmptyNodes()
+        #print ElementTree.tostring(self.xmlTree.getroot())
+        filename            = "lrt-%s.cmdi" % self.nodeId
+        self.xmlTree.write(filename, encoding = "utf-8", xml_declaration = True)
+        f                   = open(filename, 'r+' )
+        content             = f.read().replace('<CMD', '<CMD xmlns="http://www.clarin.eu/cmd/"')
+        f.close()
+        f                   = open(filename, 'w' )
+        f.write(content)
+        f.close
+
+    def addFormats(self, format):
+        if ";" in format or "," in format:
+            if ";" in format:
+                formatItems = format.split(";")
+            else:
+                formatItems = format.split(",")
+            self.fillMultipleElement("Format", "//LrtCommon/Format", formatItems)
+        else:
+            self.fillElement("//LrtCommon/Format", format)
+
+    def addInstitutes(self, institute):
+        if ";" in institute:
+                items = institute.split(";")
+                uniqueItems = set(items) # filter out double items
+                items = [i for i in uniqueItems] # convert set back to a list
+                #print items
+
+                self.fillMultipleElement("Institute", "//LrtCommon/Institute", items)
+
+    def addCountries(self, countryList, countries):
+        countriesNode = self.xmlTree.find("//LrtCommon/Countries")
+        goodList = [c.strip() for c in countries.split("||")]
+        for country in goodList:
+            if country:
+                newCountryNode = ElementTree.Element("Country")
+                newCodeNode = ElementTree.Element("Code")
+                newCodeNode.text = countryList[country]
+                newCountryNode.append(newCodeNode)
+                countriesNode.append(newCountryNode)
+
+    def addLanguages(self, isoList, languages, iso639Type = 3, xpath = "//LrtCommon/Languages"):
+        languagesNode = self.xmlTree.find(xpath)
+        languageList = [l.strip() for l in languages.split("||")]
+        for language in languageList:
+            if language and not language == "-- language not in list --":
+                newLanguageNode = ElementTree.Element("ISO639")
+                newCodeNode = ElementTree.Element("iso-639-%s-code" % iso639Type)
+                keyLang = language.encode("utf-8")
+                newCodeNode.text = isoList[keyLang]
+                newLanguageNode.append(newCodeNode)
+                languagesNode.append(newLanguageNode)
+
+
+    def addResourceType(self, types, record, isoList):
+            typeList = [t.strip() for t in types.split("||")]
+            self.fillMultipleElement("ResourceType", "//LrtCommon/ResourceType", typeList)
+            typeList = frozenset(typeList)
+
+            collectionList  = frozenset(("Spoken Corpus", "Written Corpus", "Multimodal Corpus", "Aligned Corpus", "Treebank", "N-Gram Model",))
+            lexiconList     = frozenset(("Grammar", "Lexicon / Knowledge Source", "Terminological Resource",))
+
+            if typeList.intersection(collectionList):
+                self.addCollectionDetails(record, isoList)
+            if typeList.intersection(lexiconList):
+                self.addLexiconDetails(record, isoList)
+            #if "Web Service" in typeList:
+            #    self.addServiceDetails(record)
+
+    def addCollectionDetails(self, record, isoList):
+        LrtCollectionDetails_XPath  = "//LrtInventoryResource/LrtCollectionDetails"
+
+
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/LongTermPreservationBy",    record["field_longterm_preservation"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/Location",                  record["field_location_0"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/ContentType",               record["field_content_type"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/FormatDetailed",            record["field_format_detailed"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/Quality",                   record["field_quality"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/Applications",              record["field_applications"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/Size",                      record["field_size"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/DistributionForm",          record["field_distribution_form"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/Size",                      record["field_size"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/Access",                    record["field_access"])
+        self.fillOptionalElement(LrtCollectionDetails_XPath + "/Source",                    record["field_source_0"])
+
+        # ok - this can be done in a cleaner way
+        self.addLanguages(isoList, record["field_working_languages"], 1, LrtCollectionDetails_XPath + "/WorkingLanguages")
+
+    def addLexiconDetails(self, record, isoList):
+        LrtLexiconDetails_XPath     = "//LrtInventoryResource/LrtLexiconDetails"
+
+        self.fillOptionalElement(LrtLexiconDetails_XPath + "/Date",                         record["field_date_0"])
+        self.fillOptionalElement(LrtLexiconDetails_XPath + "/Type",                         record["field_type"])
+        self.fillOptionalElement(LrtLexiconDetails_XPath + "/FormatDetailed",               record["field_format_detailed_1"])
+        self.fillOptionalElement(LrtLexiconDetails_XPath + "/SchemaReference",              record["field_schema_reference"])
+        self.fillOptionalElement(LrtLexiconDetails_XPath + "/Size",                         record["field_size_0"])
+        self.fillOptionalElement(LrtLexiconDetails_XPath + "/Access",                       record["field_access_1"])
+        self.addLanguages(isoList, record["field_working_languages_0"], 1, LrtLexiconDetails_XPath + "/WorkingLanguages")
+
+    def addServiceDetails(self, record):
+        LrtLexiconDetails_XPath  = "//LrtInventoryResource/LrtServiceDetails"
+
+        self.fillOptionalElement(LrtLexiconDetails_XPath + "/Date",                         record["field_date_0"])
+
+    def addResourceProxy(self, link) :
+        template = '''<ResourceProxy id="reflink">
+                <ResourceType>Resource</ResourceType>
+                <ResourceRef></ResourceRef>
+            </ResourceProxy>'''
+        partTree = ElementTree.XML(template)
+        parent = self.xmlTree.find(".//ResourceProxyList")
+        parent.append(partTree)
+
+        # and now fill it
+        self.fillElement("//ResourceProxy/ResourceRef", link)
+
+    def addTags(self, tags_string) :
+        tags_parent_XPath               = "//LrtInventoryResource" # One could use "/..", but that is unnecessary and can lead to mistakes.
+        tags_XML_element                = self.xmlTree.find("//LrtInventoryResource/tags")
+        assert(tags_XML_element is not None)
+
+        tags = filter(None, tags_string.split(","))
+        if len(tags) > 0 :
+            # Remove whitespace left and right to tag values
+            tags                        = list(map(unicode.strip, tags)) # X- Python 3 incompatible
+            # Remove empty strings from tags list.
+            tags                        = list(filter(None, tags))
+
+            for tag in tags :
+                tag_XML_element         = ElementTree.Element('tag')
+                tag_XML_element.text    = tag
+                tags_XML_element.append(tag_XML_element)
+        else :
+            tags_parent_element         = self.xmlTree.find(tags_parent_XPath)
+            tags_parent_element.remove(tags_XML_element)
+
+def addChildNode(parent, tag, content) :
+    node = ElementTree.Element(tag)
+    node.text = content
+    parent.append(node)
+
+
+def parseFirstLine(l):
+    keyList = [l[0].lower()]
+    for key in l[1:]:
+        if "(" in key:
+            keyList.append(key.split("(")[-1].replace(")", "").lower())
+        else:
+            keyList.append(key.replace(" ", "_").lower())
+    return keyList
+
+
+def loadInfo():
+    csvFile = csv.reader(urllib.urlopen("http://www.clarin.eu/export_resources").readlines())
+    #csvFile = csv.reader(urllib.urlopen("resources.csv").readlines())
+    #csvFile =[l.decode('utf-8') for l in rawCsvFile]
+
+
+    linenr = 0
+    newDict = dict()
+    for l in csvFile:
+        if linenr == 0:
+            fieldList = parseFirstLine(l)
+        else:
+            newDict[linenr] = dict()
+            colnr = 0
+            for field in fieldList:
+                newDict[linenr][fieldList[colnr].replace(" ", "_").decode('utf-8')] = l[colnr].decode('utf-8')
+                colnr += 1
+        linenr += 1
+    return newDict
+
+def loadCsv(filename):
+    csvFile = csv.reader(urllib.urlopen(filename).readlines())
+    dictionary = dict()
+    for l in csvFile:
+        dictionary[l[1]] = l[0]
+
+    return dictionary
+
+# only to be used in case we use the namespace, but as it is causing a lot of extra coding we just add an xmlns attribute in the end and ignore it
+#    def fixXpath(self, xpath):
+#        if xpath[0:2] == "//":
+#            xpath = "//{http://www.clarin.eu/cmd/}" + xpath[2:].replace("/", "/{http://www.clarin.eu/cmd/}")
+#        else:
+#            xpath = xpath.replace("/", "/{http://www.clarin.eu/cmd/}")
+#        return xpath
+
+def main():
+    infoDict = loadInfo()
+    countryList = loadCsv("country_codes.csv")
+    iso6393List = loadCsv("639-3-language_codes.csv")
+    iso6391List = loadCsv("639-1-language_codes.csv")
+
+    for record in infoDict.values():
+        print "creating lrt-%s.cmdi" % record["nid"]
+
+        cmdi = CmdiFile(record["nid"])
+
+        # 1-to-1 fields, easy case
+        cmdi.fillElement("//LrtCommon/ResourceName", record["name"])
+        cmdi.fillElement("//LrtCommon/Description", record["field_description"])
+        cmdi.fillElement("//LrtCommon/ContactPerson", record["field_creator"])
+        cmdi.fillElement("//LrtCommon/LanguagesOther", record["field_languages_other"])
+        cmdi.fillElement("//LrtCommon/BeginYearResourceCreation", record["field_year"])
+        cmdi.fillElement("//LrtCommon/FinalizationYearResourceCreation", record["field_end_creation_date"])
+        cmdi.fillElement("//LrtCommon/MetadataLink", record["field_metadata_link"])
+        cmdi.fillElement("//LrtCommon/Publications", record["field_publications"])
+        cmdi.fillElement("//LrtCommon/ReadilyAvailable", record["field_resource_available"].replace("Yes","true").replace("No","false"))
+        cmdi.fillElement("//LrtCommon/ReferenceLink", record["field_reference_link"])
+
+        cmdi.fillElement("//LrtDistributionClassification/DistributionType", record["distribution_type"])
+        cmdi.fillElement("//LrtDistributionClassification/ModificationsRequireRedeposition", record["modifications_require_redeposition"].replace("1","true").replace("0","false"))
+        cmdi.fillElement("//LrtDistributionClassification/NonCommercialUsageOnly", record["non-commercial_usage_only"].replace("1","true").replace("0","false"))
+        cmdi.fillElement("//LrtDistributionClassification/UsageReportRequired", record["usage_report_required"].replace("1","true").replace("0","false"))
+        cmdi.fillElement("//LrtDistributionClassification/OtherDistributionRestrictions", record["other_distribution_restrictions"])
+
+        cmdi.fillElement("//LrtIPR/EthicalReference", record["field_ethical_reference"])
+        cmdi.fillElement("//LrtIPR/LegalReference", record["field_legal_reference"])
+        cmdi.fillElement("//LrtIPR/LicenseType", record["field_license_type"])
+        cmdi.fillElement("//LrtIPR/Description", record["field_description_0"])
+        cmdi.fillElement("//LrtIPR/ContactPerson", record["field_contact_person"])
+
+        # add a ResourceProxy for ReferenceLink
+        if "http" in record["field_reference_link"]:
+            cmdi.addResourceProxy(record["field_reference_link"])
+
+        # more sophisticated (dirty) tricks needed
+        cmdi.addFormats(record["field_format"])
+
+        orgList = ""
+        for i in range(1,5):
+            orgList += record["org" + str(i)] + ";"
+        cmdi.addInstitutes(orgList + record["field_institute"])
+
+        cmdi.addCountries(countryList, record["field_country"])
+
+        cmdi.addLanguages(iso6393List, record["field_languages"])
+
+        cmdi.addResourceType(record["field_resource_type"], record, iso6391List)
+
+        cmdi.addTags(record['tags']);
+
+        cmdi.serialize()
+
+main()
