Index: MDRepository/trunk/xquery/README
===================================================================
--- MDRepository/trunk/xquery/README	(revision 959)
+++ MDRepository/trunk/xquery/README	(revision 1045)
@@ -12,5 +12,12 @@
 	c) set admin pwd 
 	
-	...?
+	d) you may want to add memory to the JVM
+	   under bin/functions.d/eXist-settings.sh#set_java_options()
+	
+	e) you may also want to grow the cache in conf.xml
+		 <db-connection cacheSize="48M" collectionCache="24M" database="native"        
+		  where @cacheSize could be around 512M 
+      and @collectionCache should be around one third of the @cacheSize
+
 
 1. add scripts to: /db/clarin
@@ -18,17 +25,9 @@
 		+ cmd-model.xqm has all the logic
 		+ cmd-model.xql is the script being called as the interface
-		+ cmd-stats.xql is meant for testing purposes, but not integrated yet
-		+ init-cache.xql is meant for refreshing the cache with some long-running (resource-intensive) queries, meant to run once upon dataset change
+	 (+) cmd-stats.xql is meant for testing purposes, but not integrated yet
+	 (+) init-cache.xql is meant for refreshing the cache with some long-running (resource-intensive) queries, meant to run once upon dataset change
 
-2. add data to  /db/cmdi-mirror
-	 (the file-system structure will be reflected in the "collection"-structure within exist,
-	 however this is irrelevant for the MDRepository methods.
-	 Those rely on the linking via handles in MdSelfLink/ResourceRef and IsPartOf elements of the MDRecords.
-	 The handles in IsPartOf are redundant (necessary for faster collection-constraint search) 
-	 and can be derived from the ResourceRef/MdSelfLink link.
-	 This can be done before storing the data in the repository,
-	 or after the import directly in the repository (scripts for this will be available soon)
 
-3. add a clarin-user in /db/system/users.xml
+2. add a clarin-user in /db/system/users.xml
    (needed for writing into the cache)
    + /db/clarin/writer.xml with given user, like this:
@@ -39,8 +38,8 @@
 
 
-4. create a collection for caching, 
-	eg: /db/common/clarin/freqs
+3. create a collection for caching, 
+	eg: /db/cache
 	this has to correspond to the entry in cmd-model.xqm:
-	declare variable $cmd-model:commonFreqsPath as xs:string := "/db/common/clarin/freqs";
+	declare variable $cmd-model:commonFreqsPath as xs:string := "/db/cache";
 	
 	If you change something, you have to manually clear the cache-collection.
@@ -50,6 +49,24 @@
 	  for getCollections: collection{maxdepth}-{hash({collection-handle})}
 	  for queryModel:   values{maxdepth}-{hash({simple xpath from q-param})}
+
 	
-5. depending on your server-setup you should be able to get your first query under somewhere like:		
+4. define indices 
+	 copy cmdi-mirror.xconf into /db/system/config/db/cmdi-mirror
+	  
+	  
+5. add data to  /db/cmdi-mirror
+	 (the file-system structure will be reflected in the "collection"-structure within exist,
+	 however this is irrelevant for the MDRepository methods.
+	 Those rely on the linking via handles in MdSelfLink/ResourceRef and <IsPartOf> elements of the MDRecords.
+	 The handles in <IsPartOf> are redundant (necessary for faster collection-constraint search) 
+	 and can be derived from the ResourceRef/MdSelfLink link.
+	 This can be done before storing the data in the repository,
+	 or after the import directly in the repository (XUpdate-scripts for this will be available soon)
+
+	 The top level collection record is by convention called colleciton_root.cmdi
+	 and is marked with: <IsPartOf>root</IsPartOf>
+	 (So every dataset (olac, lrt, imdi) has one such MDRecord.)
+
+6. depending on your server-setup (port) you should be able to get your first query under somewhere like:		
 		
 	http://localhost:8680/exist/rest/db/clarin/cmd-model.xql?q=Components
@@ -61,5 +78,6 @@
 	Avoid starting multiple times.
 	You can see in the cache-collection, if the results are ready.
-	
+
+
 	
 == test suite ==
Index: MDRepository/trunk/xquery/build-tests.properties
===================================================================
--- MDRepository/trunk/xquery/build-tests.properties	(revision 959)
+++ MDRepository/trunk/xquery/build-tests.properties	(revision 1045)
@@ -2,2 +2,4 @@
 test.file=${cmdi-tests.basedir}/cmd-test.xml
 tests.output = ${cmdi-tests.basedir}/output
+//test.group = cmdi-tests-searchRetrieve
+test.group = cmdi-tests-cc
Index: MDRepository/trunk/xquery/build-tests.xml
===================================================================
--- MDRepository/trunk/xquery/build-tests.xml	(revision 959)
+++ MDRepository/trunk/xquery/build-tests.xml	(revision 1045)
@@ -43,5 +43,5 @@
         </typedef>
 				
-				<echo message="queries-file:${test.file}"/> 
+				<echo message="queries-file#group:${test.file}#${test.group}"/> 				
         <delete dir="${tests.output}/temp" failonerror="false"/>
         <mkdir dir="${tests.output}"/>
@@ -50,5 +50,5 @@
        <test:benchmark outputFile="${tests.output}/cmdi-result.xml"
                 source="${test.file}"
-                group="cmdi-tests-cc"/>        
+                group="${test.group}"/>        
     </target>
 
Index: MDRepository/trunk/xquery/cmd-model.xql
===================================================================
--- MDRepository/trunk/xquery/cmd-model.xql	(revision 959)
+++ MDRepository/trunk/xquery/cmd-model.xql	(revision 1045)
@@ -29,13 +29,19 @@
     return
       if ($operation eq $cmd-model:getCollections) then
-	cmd-model:get-collections($query-collections, $format, $max-depth)
+		cmd-model:get-collections($query-collections, $format, $max-depth)
       else if ($operation eq $cmd-model:queryModel) then
-	cmd-model:query-model($cmd-index-path, $query-collections, $format, $max-depth)
-    else if ($operation eq $cmd-model:searchRetrieve) then
-      let $cql-query := request:get-parameter("query", "MDGroup/Actors/Actor"),
-	$start-item := request:get-parameter("startRecord", 1),
-	$end-item := request:get-parameter("iend", 50)
+		cmd-model:query-model($cmd-index-path, $query-collections, $format, $max-depth)
+      else if ($operation eq $cmd-model:scanIndex) then
+		let $filter := request:get-parameter("filter", ""),
+		$start-item := request:get-parameter("startItem", 1),
+		$max-items := request:get-parameter("maxItems", 50),
+		$sort := request:get-parameter("sort", 'text')
+		return cmd-model:scan-index($cmd-index-path, $query-collections, $format, $start-item, $max-items, $sort)
+	  else if ($operation eq $cmd-model:searchRetrieve) then
+      	let $cql-query := request:get-parameter("query", "MDGroup/Actors/Actor"),
+			$start-item := request:get-parameter("startItem", 1),
+			$max-items := request:get-parameter("maxItems", 50)
 	
-      return cmd-model:search-retrieve($cql-query, $query-collections, $format, xs:integer($start-item), xs:integer($end-item))
+      return cmd-model:search-retrieve($cql-query, $query-collections, $format, xs:integer($start-item), xs:integer($max-items))
     else 
       <error>Unknown operation</error>
Index: MDRepository/trunk/xquery/cmd-model.xqm
===================================================================
--- MDRepository/trunk/xquery/cmd-model.xqm	(revision 959)
+++ MDRepository/trunk/xquery/cmd-model.xqm	(revision 1045)
@@ -12,7 +12,8 @@
 declare variable $cmd-model:cmdiMirrorPath as xs:string := "/db/cmdi-mirror";
 declare variable $cmd-model:cachePath as xs:string := "/db/cache";
-
+declare variable $cmd-model:groupXsl := doc('/db/clarin/group.xsl');
 declare variable $cmd-model:getCollections as xs:string := "getCollections";
 declare variable $cmd-model:queryModel as xs:string := "queryModel";
+declare variable $cmd-model:scanIndex as xs:string := "scanIndex";
 declare variable $cmd-model:searchRetrieve as xs:string := "searchRetrieve";
 
@@ -28,4 +29,7 @@
 declare variable $cmd-model:responseFormatText as xs:string := "text";
 
+declare variable $cmd-model:scanSortText as xs:string := "text";
+declare variable $cmd-model:scanSortSize as xs:string := "size";
+
 declare variable $cmd-model:collectionDocName as xs:string := "collection.xml";
 
@@ -34,6 +38,22 @@
 declare variable $cmd-model:xmlExt as xs:string := ".xml";
 
+declare variable $cmd-model:maxDepth as xs:integer := 8;
 declare variable $cmd-model:valuesLimit as xs:integer := 100;
 
+
+(:~
+  API function getCollections. 
+:)
+declare function cmd-model:get-collections($collections as xs:string+, $format as xs:string, $max-depth as xs:integer) as item() {
+  let $name := cmd-model:gen-cache-id("collection", $collections, xs:string($max-depth)),
+    $doc := 
+    if (cmd-model:is-in-cache($name)) then
+       cmd-model:get-from-cache($name)
+    else
+      let $data := cmd-model:colls($collections, $max-depth)
+ 	return cmd-model:store-in-cache($name, $data)
+  return 
+    cmd-model:serialise-as($doc, $format)
+};
 
 (:~
@@ -54,16 +74,45 @@
 
 (:~
-  API function getCollections. 
-:)
-declare function cmd-model:get-collections($collections as xs:string+, $format as xs:string, $max-depth as xs:integer) as item() {
-  let $name := cmd-model:gen-cache-id("collection", $collections, xs:string($max-depth)),
-    $doc := 
-    if (cmd-model:is-in-cache($name)) then
-       cmd-model:get-from-cache($name)
-    else
-      let $data := cmd-model:colls($collections, $max-depth)
- 	return cmd-model:store-in-cache($name, $data)
+  API function scanIndex. 
+two phases: 
+	1.one create full index for given path/element (and cache)
+	2. select wished subsequence (on second call, only the second step is performed)
+:)
+declare function cmd-model:scan-index($q as xs:string, $collection as xs:string+, $format as xs:string, $start-item as xs:integer, $max-items as xs:integer, $p-sort as xs:string?) as item()? {
+
+  let $qa := tokenize($q,'='),
+	 $cmd-index-path := $qa[1],
+ 	 $filter := ($qa[2],'')[1],
+	 $sort := if ($p-sort eq $cmd-model:scanSortText or $p-sort eq $cmd-model:scanSortSize) then $p-sort else $cmd-model:scanSortText,
+  	  $name := cmd-model:gen-cache-id("index", ($collection, $cmd-index-path),"1"),
+    (: skip cache $doc := cmd-model:values($cmd-index-path, $collection) :)
+    $doc := if (cmd-model:is-in-cache($name)) then
+      cmd-model:get-from-cache($name) 
+    else  
+      let  $data := cmd-model:values($cmd-index-path, $collection)
+        return cmd-model:store-in-cache($name, $data)
+
+	(: extract the required subsequence (according to given sort) :)
+	let $res-term := transform:transform($doc,$cmd-model:groupXsl, 
+			<parameters><param name="mode" value="subsequence"/>
+						<param name="sort" value="{$sort}"/>
+						<param name="filter" value="{$filter}"/>
+						<param name="start-item" value="{$start-item}"/>
+						<param name="max-items" value="{$max-items}"/>
+			</parameters>),
+		$count-items := count($res-term/v),
+		$colls := if (fn:empty($collection)) then '' else fn:string-join($collection, ","),
+		$created := fn:current-dateTime(),
+		$scan-clause := concat($cmd-index-path, '=', $filter),
+		$res := <Terms colls="{$colls}" created="{$created}" count_items="{$count-items}" 
+					start-item="{$start-item}" max-items="{$max-items}" sort="{$sort}" scanClause="{$scan-clause}"  >{$res-term}</Terms>
+
+(:	let	$result-count := $doc/Term/@count,
+    $result-seq := fn:subsequence($doc/Term/v, $start-item, $end-item),
+	$result-frag := ($doc/Term, $result-seq),
+    $seq-count := fn:count($result-seq) :)
+
   return 
-    cmd-model:serialise-as($doc, $format)
+    cmd-model:serialise-as($res, $format)	
 };
 
@@ -82,9 +131,22 @@
       for $coll in $collections return util:eval(fn:concat("$collection/ft:query(descendant::IsPartOf, <term>", xdb:decode($coll) ,"</term>)/ancestor-or-self::CMD", $sanitized-query))
 
-    let $result-count := fn:count($results),
+	let	$result-count := fn:count($results),
     $result-seq := fn:subsequence($results, $start-item, $end-item),
     $seq-count := fn:count($result-seq),
-    $end-time := util:system-dateTime(),
-    $result-fragment :=
+	$end-time := util:system-dateTime()
+
+	let $summary-fragment :=
+		if (contains($format,'withSummary')) then
+			let $used-profiles := for $profile in distinct-values($results//Components/concat(child::element()/name(),'##',../Header/MdProfile))
+	 						let $profile-id := substring-after($profile,'##'), $profile-name := substring-before($profile,'##')
+							return <profile id="{$profile-id}" name="{$profile-name}" count="{count($results//Components[concat(child::element()/name(),'##',../Header/MdProfile) eq $profile])}" />,
+				$end-time2 := util:system-dateTime(),
+				$result-summary := cmd-model:elem-r($result-seq//Components, "Components", $cmd-model:maxDepth, $cmd-model:maxDepth),
+		    	$end-time3 := util:system-dateTime(),
+				$duration :=  concat(($end-time - $start-time),", ", ($end-time2 - $start-time),", ", ($end-time3 - $start-time))
+			return (<duration>{$duration}</duration>, <usedProfiles>{$used-profiles}</usedProfiles>,<resultSummary>{$result-summary}</resultSummary>)
+		else <duration>{$end-time - $start-time}</duration>
+		
+    let $result-fragment :=
     <searchRetrieveResponse>
       <numberOfRecords>{$result-count}</numberOfRecords>
@@ -92,5 +154,5 @@
       <extraResponseData>
       	<returnedRecords>{$seq-count}</returnedRecords>
-      	<duration>{$end-time - $start-time}</duration>
+		{$summary-fragment}
       </extraResponseData>
       <records>
@@ -106,5 +168,5 @@
 (: 
   **********************
-  queryModel - subfunctions
+  queryModel, scanIndex - subfunctions
 :)
 
@@ -154,20 +216,10 @@
 	    (for $elname in $subs[. != '']
 	    return
-	      cmd-model:elem-r(util:eval(concat("$path-nodes/", $elname)), concat($path, '/', $elname), $max-depth, $depth - 1),
-	      if ($max-depth eq 1 and $text-count gt 0) then cmd-model:values($path-nodes) else ())
+	      cmd-model:elem-r(util:eval(concat("$path-nodes/", $elname)), concat($path, '/', $elname), $max-depth, $depth - 1)
+			(: values moved to own function: scanIndex 
+		      if ($max-depth eq 1 and $text-count gt 0) then cmd-model:values($path-nodes) else ()) :)
+						)
 	  else 'maxdepth'
 	}</Term>
-};
-
-declare function cmd-model:values($nodes as node()*) as node()* {
-let $keys := distinct-values($nodes/text())
-let $values := for $key at $pos in $keys
-  let $kcount := count($nodes[. eq $key])
-    order by lower-case($key) ascending
-    return <v key="{$key}" cnt="{$kcount}" />
-return
-  if ($cmd-model:valuesLimit eq 0) then $values 
-  else
-  subsequence($values, 1, $cmd-model:valuesLimit)
 };
 
@@ -178,4 +230,29 @@
 	return util:node-xpath($anc)
 	}</Term>
+};
+
+declare function cmd-model:collect-nodes($collections as xs:string+, $path as xs:string) as element()* {
+  let $collection := collection($cmd-model:cmdiMirrorPath),
+    $path-nodes :=
+    if ($collections[1] eq $cmd-model:collectionRoot) then
+      util:eval(fn:concat("$collection/descendant-or-self::", $path))
+    else
+      for $coll in $collections 
+      return
+	util:eval(fn:concat("$collection/ft:query(descendant::IsPartOf, <query><term>", xdb:decode($coll), "</term></query>)/ancestor-or-self::CMD/descendant-or-self::", $path))
+   
+	return $path-nodes
+};
+
+declare function cmd-model:values($path as xs:string,$collections as xs:string+) as element() {
+
+ 	let $nodes := cmd-model:collect-nodes($collections, $path),
+(:		$term := <Term path="{fn:concat("//", $path)}" name="{(text:groups($path, "/([^/]+)$")[last()],$path)[1] }" >{$nodes}</Term> 
+		@name is added in xslt:)
+		$term := <Term path="{fn:concat("//", $path)}"  >{$nodes}</Term>
+
+	(: use XSLT-2.0 for-each-group functionality to aggregate the values of a node - much, much faster, than XQuery :)
+	return transform:transform($term,$cmd-model:groupXsl, ())
+
 };
 
Index: MDRepository/trunk/xquery/cmd-test.xml
===================================================================
--- MDRepository/trunk/xquery/cmd-test.xml	(revision 959)
+++ MDRepository/trunk/xquery/cmd-test.xml	(revision 1045)
@@ -3,5 +3,6 @@
     <configuration>
         <!--  <connection id="con" user="admin" password="" base="xmldb:exist://embedded-eXist-server"/> -->
-        <connection id="con" user="admin" password="pwd" base="xmldb:exist://localhost:8680/exist/xmlrpc/"/>
+        <!-- <connection id="con" user="admin" password="M0dest7db" base="xmldb:exist://localhost:8680/exist/xmlrpc"/> -->
+        <connection id="con" user="admin" password="sen71blE8dba" base="xmldb:exist://clarin.aac.ac.at/exist/xmlrpc"/> 
         <action name="sequence" class="org.exist.performance.ActionSequence"/>
         <action name="create-collection" class="org.exist.performance.actions.CreateCollection"/>
@@ -40,16 +41,16 @@
 				<thread name="thread1" connection="con">
             <sequence repeat="20" description="actual queries on the mdrecords">	      
-	      				<xquery collection="/db/cmdi-mirror" query="">
+	      				<xquery collection="db/cmdi-mirror" query="">
 	      					import module namespace cmd-model = "http://spraakbanken.gu.se/clarin/xquery/model"
 									at "xmldb:exist:///db/clarin/cmd-model.xqm";
 	      					 cmd-model:search-retrieve("//title[contains(.,'a')]", 'root', 'xml', 1,10)
 	      				</xquery>
-	      				<xquery collection="/db/cmdi-mirror" query="//ft:query(*, <query><term>Language</term></query>)" />
-	      				<xquery collection="/db/cmdi-mirror" query="ft:query(descendant::IsPartOf, <query><term>clarin-at:aac-test-corpus</term></query>)" />
+	      				<xquery collection="/db/cmdi-mirror" query="//Components/ft:query(., 'Language')" />
+	      				<xquery collection="/db/cmdi-mirror" query="ft:query(descendant::IsPartOf, 'clarin-at:aac-test-corpus')" />
 	      				<xquery collection="/db/cmdi-mirror" query="//idno[.='4197']" />
 	      				<xquery collection="/db/cmdi-mirror" query="//date[.>'1920']" />
 	      				<xquery collection="/db/cmdi-mirror" query="//ResourceType[.='Metadata']" />
 	      				<xquery collection="/db/cmdi-mirror" query="//sourceDesc/biblStruct/monogr/title[contains(.,'rat')]" />
-	      				<xquery collection="/db/cmdi-mirror" query="//CMD/*[.//teiHeader//monogr/title[contains(.,'t')]][.//teiHeader//imprint/date[.>'1930']][.//teiHeader//imprint/date[.<'1935']]" />	      				
+	      				<xquery collection="/db/cmdi-mirror" query="//CMD/*[.//teiHeader//monogr/title[contains(.,'t')]][.//teiHeader//imprint/date[.&gt;'1930']][.//teiHeader//imprint/date[.&lt;'1935']]" />	      				
 	      		</sequence>
 				</thread>
Index: MDRepository/trunk/xquery/cmdi-mirror.xconf
===================================================================
--- MDRepository/trunk/xquery/cmdi-mirror.xconf	(revision 1045)
+++ MDRepository/trunk/xquery/cmdi-mirror.xconf	(revision 1045)
@@ -0,0 +1,11 @@
+<collection xmlns="http://exist-db.org/collection-config/1.0">
+    <index>
+        <lucene>
+            <analyzer class="org.apache.lucene.analysis.WhitespaceAnalyzer"/>			
+            <text qname="MdSelfLink"/>
+            <text qname="IsPartOf"/>
+            <text qname="ResourceRef"/>
+            <text qname="Components"/>
+        </lucene>      
+    </index>
+</collection>
Index: MDRepository/trunk/xquery/group.xsl
===================================================================
--- MDRepository/trunk/xquery/group.xsl	(revision 1045)
+++ MDRepository/trunk/xquery/group.xsl	(revision 1045)
@@ -0,0 +1,84 @@
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" 
+ xmlns:xs="http://www.w3.org/2001/XMLSchema"
+version="2.0">
+
+<!--
+two tasks (in separate calls, managed by $mode-param):
+1. produces an index by grouping content	
+2. selects a subsequence of the produced content
+-->
+
+<xsl:param name="mode" select="'aggregate'" />
+<xsl:param name="sort" select="'text'" />
+<xsl:param name="filter" select="''" />
+<xsl:param name="filter-mode" select="if (ends-with($filter,'*')) then 'starts-with' else 'contains'" /> <!-- contains, starts-with -->
+<xsl:param name="start-item" select="100" />
+<xsl:param name="max-items" select="100" />
+
+    <xsl:template match="/">
+<!--		<Terms>
+			<params mode="{$mode}" sort="{$sort}"  /> -->
+		<xsl:choose>
+			<xsl:when test="$mode='subsequence'" >
+				<xsl:apply-templates mode="subsequence" />
+			</xsl:when>
+			<xsl:otherwise>
+				<xsl:apply-templates />
+			</xsl:otherwise>
+		</xsl:choose>
+    </xsl:template>
+
+    <xsl:template match="Term">
+		<xsl:variable name="nodes" select="*" />
+		<xsl:variable name="count-text" select="count($nodes/text()[.!=''])" />
+		<xsl:variable name="distinct-text-count" select="count(distinct-values($nodes/text()))" />
+
+		<Term name="{*[1]/name()}" count="{count($nodes)}" count_text="{$count-text}"  count_distinct_text="{$distinct-text-count}">
+			<xsl:copy-of select="@*" />
+            <xsl:for-each-group select="*" group-by="text()">
+                <v key="{text()}" count="{count(current-group())}"/>
+            </xsl:for-each-group>
+        </Term>
+    </xsl:template>
+
+    <xsl:template match="Term" mode="subsequence">
+
+	<!-- this may be potentially expensive and we may need to store the index already sorted (which is the normal/sane way to do!) -->
+		<xsl:variable name="filtered" select="*[if ($filter!='') then 
+														if ($filter-mode='starts-with') then starts-with(@key,substring-before($filter,'*'))
+ 																	else contains(@key, $filter)
+														else true()]" />
+		<xsl:variable name="ordered" >
+			<xsl:choose>
+			<xsl:when test="$sort='size'" >	
+				<xsl:for-each select="$filtered" >
+					<xsl:sort select="@count" data-type="number" order="descending" />
+					<xsl:copy-of select="." />
+				</xsl:for-each>
+			</xsl:when>
+			<xsl:otherwise>
+				<xsl:for-each select="$filtered" >
+					<xsl:sort select="@key" data-type="text" order="ascending" />					
+					<xsl:copy-of select="."/>
+				</xsl:for-each>
+			</xsl:otherwise>
+			</xsl:choose>
+		</xsl:variable>
+
+<!-- 		<xsl:variable name="count-items" select="count($filtered)" />-->
+		<xsl:copy>
+			<xsl:copy-of select="@*" />
+<!--			<xsl:value-of select="count($ordered/*)" /> -->
+<!--			<xsl:attribute name="count_items" select="if (xs:integer($count-items) &gt; xs:integer($max-items)) then $max-items else $count-items" /> -->
+				<xsl:apply-templates select="$ordered/*[xs:integer(position()) &gt;= xs:integer($start-item) and (xs:integer(position()) &lt; (xs:integer($start-item) + xs:integer($max-items)))]" />
+		</xsl:copy>
+    </xsl:template>
+
+	<xsl:template match="v" >
+		<xsl:copy>
+			<xsl:copy-of select="@*" />
+			<xsl:attribute name="pos" select="position()" />
+		</xsl:copy>
+	</xsl:template>
+
+</xsl:stylesheet>
