Changeset 1045 for MDRepository/trunk/xquery/cmd-model.xqm

Timestamp:

01/09/11 20:28:15 (13 years ago)

Author:

vronk

Message:

mainly added new API-function: scanIndex
(group.xsl used for aggregating)

File:

• MDRepository/trunk/xquery/cmd-model.xqm (modified) (9 diffs)

MDRepository/trunk/xquery/cmd-model.xqm

@@ -12 +12 @@
 declare variable $cmd-model:cmdiMirrorPath as xs:string := "/db/cmdi-mirror";
 declare variable $cmd-model:cachePath as xs:string := "/db/cache";
-
+declare variable $cmd-model:groupXsl := doc('/db/clarin/group.xsl');
 declare variable $cmd-model:getCollections as xs:string := "getCollections";
 declare variable $cmd-model:queryModel as xs:string := "queryModel";
+declare variable $cmd-model:scanIndex as xs:string := "scanIndex";
 declare variable $cmd-model:searchRetrieve as xs:string := "searchRetrieve";
 
@@ -28 +29 @@
 declare variable $cmd-model:responseFormatText as xs:string := "text";
 
+declare variable $cmd-model:scanSortText as xs:string := "text";
+declare variable $cmd-model:scanSortSize as xs:string := "size";
+
 declare variable $cmd-model:collectionDocName as xs:string := "collection.xml";
 
@@ -34 +38 @@
 declare variable $cmd-model:xmlExt as xs:string := ".xml";
 
+declare variable $cmd-model:maxDepth as xs:integer := 8;
 declare variable $cmd-model:valuesLimit as xs:integer := 100;
 
+
+(:~
+  API function getCollections. 
+:)
+declare function cmd-model:get-collections($collections as xs:string+, $format as xs:string, $max-depth as xs:integer) as item() {
+  let $name := cmd-model:gen-cache-id("collection", $collections, xs:string($max-depth)),
+    $doc := 
+    if (cmd-model:is-in-cache($name)) then
+       cmd-model:get-from-cache($name)
+    else
+      let $data := cmd-model:colls($collections, $max-depth)
+        return cmd-model:store-in-cache($name, $data)
+  return 
+    cmd-model:serialise-as($doc, $format)
+};
 
 (:~
@@ -54 +74 @@
 
 (:~
-  API function getCollections. 
-:)
-declare function cmd-model:get-collections($collections as xs:string+, $format as xs:string, $max-depth as xs:integer) as item() {
-  let $name := cmd-model:gen-cache-id("collection", $collections, xs:string($max-depth)),
-    $doc := 
-    if (cmd-model:is-in-cache($name)) then
-       cmd-model:get-from-cache($name)
-    else
-      let $data := cmd-model:colls($collections, $max-depth)
-        return cmd-model:store-in-cache($name, $data)
+  API function scanIndex. 
+two phases: 
+        1.one create full index for given path/element (and cache)
+        2. select wished subsequence (on second call, only the second step is performed)
+:)
+declare function cmd-model:scan-index($q as xs:string, $collection as xs:string+, $format as xs:string, $start-item as xs:integer, $max-items as xs:integer, $p-sort as xs:string?) as item()? {
+
+  let $qa := tokenize($q,'='),
+         $cmd-index-path := $qa[1],
+         $filter := ($qa[2],'')[1],
+         $sort := if ($p-sort eq $cmd-model:scanSortText or $p-sort eq $cmd-model:scanSortSize) then $p-sort else $cmd-model:scanSortText,
+          $name := cmd-model:gen-cache-id("index", ($collection, $cmd-index-path),"1"),
+    (: skip cache $doc := cmd-model:values($cmd-index-path, $collection) :)
+    $doc := if (cmd-model:is-in-cache($name)) then
+      cmd-model:get-from-cache($name) 
+    else  
+      let  $data := cmd-model:values($cmd-index-path, $collection)
+        return cmd-model:store-in-cache($name, $data)
+
+        (: extract the required subsequence (according to given sort) :)
+        let $res-term := transform:transform($doc,$cmd-model:groupXsl, 
+                        <parameters><param name="mode" value="subsequence"/>
+                                                <param name="sort" value="{$sort}"/>
+                                                <param name="filter" value="{$filter}"/>
+                                                <param name="start-item" value="{$start-item}"/>
+                                                <param name="max-items" value="{$max-items}"/>
+                        </parameters>),
+                $count-items := count($res-term/v),
+                $colls := if (fn:empty($collection)) then '' else fn:string-join($collection, ","),
+                $created := fn:current-dateTime(),
+                $scan-clause := concat($cmd-index-path, '=', $filter),
+                $res := <Terms colls="{$colls}" created="{$created}" count_items="{$count-items}" 
+                                        start-item="{$start-item}" max-items="{$max-items}" sort="{$sort}" scanClause="{$scan-clause}"  >{$res-term}</Terms>
+
+(:      let     $result-count := $doc/Term/@count,
+    $result-seq := fn:subsequence($doc/Term/v, $start-item, $end-item),
+        $result-frag := ($doc/Term, $result-seq),
+    $seq-count := fn:count($result-seq) :)
+
   return 
-    cmd-model:serialise-as($doc, $format)
+    cmd-model:serialise-as($res, $format)       
 };
 
@@ -82 +131 @@
       for $coll in $collections return util:eval(fn:concat("$collection/ft:query(descendant::IsPartOf, <term>", xdb:decode($coll) ,"</term>)/ancestor-or-self::CMD", $sanitized-query))
 
-    let $result-count := fn:count($results),
+        let     $result-count := fn:count($results),
     $result-seq := fn:subsequence($results, $start-item, $end-item),
     $seq-count := fn:count($result-seq),
-    $end-time := util:system-dateTime(),
-    $result-fragment :=
+        $end-time := util:system-dateTime()
+
+        let $summary-fragment :=
+                if (contains($format,'withSummary')) then
+                        let $used-profiles := for $profile in distinct-values($results//Components/concat(child::element()/name(),'##',../Header/MdProfile))
+                                                        let $profile-id := substring-after($profile,'##'), $profile-name := substring-before($profile,'##')
+                                                        return <profile id="{$profile-id}" name="{$profile-name}" count="{count($results//Components[concat(child::element()/name(),'##',../Header/MdProfile) eq $profile])}" />,
+                                $end-time2 := util:system-dateTime(),
+                                $result-summary := cmd-model:elem-r($result-seq//Components, "Components", $cmd-model:maxDepth, $cmd-model:maxDepth),
+                        $end-time3 := util:system-dateTime(),
+                                $duration :=  concat(($end-time - $start-time),", ", ($end-time2 - $start-time),", ", ($end-time3 - $start-time))
+                        return (<duration>{$duration}</duration>, <usedProfiles>{$used-profiles}</usedProfiles>,<resultSummary>{$result-summary}</resultSummary>)
+                else <duration>{$end-time - $start-time}</duration>
+                
+    let $result-fragment :=
     <searchRetrieveResponse>
       <numberOfRecords>{$result-count}</numberOfRecords>
@@ -92 +154 @@
       <extraResponseData>
         <returnedRecords>{$seq-count}</returnedRecords>
-        <duration>{$end-time - $start-time}</duration>
+                {$summary-fragment}
       </extraResponseData>
       <records>
@@ -106 +168 @@
 (: 
   **********************
-  queryModel - subfunctions
+  queryModel, scanIndex - subfunctions
 :)
 
@@ -154 +216 @@
             (for $elname in $subs[. != '']
             return
-              cmd-model:elem-r(util:eval(concat("$path-nodes/", $elname)), concat($path, '/', $elname), $max-depth, $depth - 1),
-              if ($max-depth eq 1 and $text-count gt 0) then cmd-model:values($path-nodes) else ())
+              cmd-model:elem-r(util:eval(concat("$path-nodes/", $elname)), concat($path, '/', $elname), $max-depth, $depth - 1)
+                        (: values moved to own function: scanIndex 
+                      if ($max-depth eq 1 and $text-count gt 0) then cmd-model:values($path-nodes) else ()) :)
+                                                )
           else 'maxdepth'
         }</Term>
-};
-
-declare function cmd-model:values($nodes as node()*) as node()* {
-let $keys := distinct-values($nodes/text())
-let $values := for $key at $pos in $keys
-  let $kcount := count($nodes[. eq $key])
-    order by lower-case($key) ascending
-    return <v key="{$key}" cnt="{$kcount}" />
-return
-  if ($cmd-model:valuesLimit eq 0) then $values 
-  else
-  subsequence($values, 1, $cmd-model:valuesLimit)
 };
 
@@ -178 +230 @@
         return util:node-xpath($anc)
         }</Term>
+};
+
+declare function cmd-model:collect-nodes($collections as xs:string+, $path as xs:string) as element()* {
+  let $collection := collection($cmd-model:cmdiMirrorPath),
+    $path-nodes :=
+    if ($collections[1] eq $cmd-model:collectionRoot) then
+      util:eval(fn:concat("$collection/descendant-or-self::", $path))
+    else
+      for $coll in $collections 
+      return
+        util:eval(fn:concat("$collection/ft:query(descendant::IsPartOf, <query><term>", xdb:decode($coll), "</term></query>)/ancestor-or-self::CMD/descendant-or-self::", $path))
+   
+        return $path-nodes
+};
+
+declare function cmd-model:values($path as xs:string,$collections as xs:string+) as element() {
+
+        let $nodes := cmd-model:collect-nodes($collections, $path),
+(:              $term := <Term path="{fn:concat("//", $path)}" name="{(text:groups($path, "/([^/]+)$")[last()],$path)[1] }" >{$nodes}</Term> 
+                @name is added in xslt:)
+                $term := <Term path="{fn:concat("//", $path)}"  >{$nodes}</Term>
+
+        (: use XSLT-2.0 for-each-group functionality to aggregate the values of a node - much, much faster, than XQuery :)
+        return transform:transform($term,$cmd-model:groupXsl, ())
+
 };