Changeset 825 for MDRepository


Ignore:
Timestamp:
11/01/10 12:50:32 (14 years ago)
Author:
vronk
Message:

caching refactored

File:
1 edited

Legend:

Unmodified
Added
Removed

  • MDRepository/trunk/xquery/cmd-model.xqm

    r803 r825  
    1010declare variable $cmd-model:cmdiDatabaseURI as xs:string := "xmldb:exist:///db";
    1111
    12 declare variable $cmd-model:commonFreqsPath as xs:string := "/db/common/clarin/freqs";
    1312declare variable $cmd-model:cmdiMirrorPath as xs:string := "/db/cmdi-mirror";
     13declare variable $cmd-model:cachePath as xs:string := "/db/cache";
    1414
    1515declare variable $cmd-model:getCollections as xs:string := "getCollections";
    … …  
    4141:)
    4242declare function cmd-model:query-model($cmd-index-path as xs:string, $collection as xs:string+, $format as xs:string, $max-depth as xs:integer) as item()? {
    43         cmd-model:serialise-as(cmd-model:get-result-doc($collection, $cmd-index-path, $max-depth), $format)
     43       
     44        let $name := cmd-model:gen-cache-id("model", ($collection, $cmd-index-path), xs:string($max-depth)),
     45        $doc :=
     46    if (cmd-model:is-in-cache($name)) then
     47      cmd-model:get-from-cache($name)
     48    else
     49        let $data := cmd-model:elem($collection, $cmd-index-path, $max-depth)
     50                return cmd-model:store-in-cache($name, $data)
     51
     52        return
     53        cmd-model:serialise-as($doc, $format)   
    4454};
    4555
    … …  
    4858:)
    4959declare function cmd-model:get-collections($collections as xs:string+, $format as xs:string, $max-depth as xs:integer) as item() {
    50   let $names := ($cmd-model:commonFreqsPath, cmd-model:make-compound-doc-name($collections, "collection", xs:string($max-depth))),
    51         $dummy :=
    52       if (cmd-model:is-doc-available($names[1], $names[2])) then
    53         ()
    54       else
    55         let $children :=
    56           for $collection-item in $collections
    57             return
    58             for $collection-doc in cmd-model:get-resource-by-handle($collection-item)
    59               return cmd-model:recurse-collections($collection-doc, cmd-model:get-md-collection-name($collection-doc), $collection-doc//MdSelfLink, "", $max-depth)
    60             let $res-count := sum($children/@cnt)
    61             let $coll-count := sum($children/@cnt_subcolls) + count($children)
    62               return cmd-model:store-collection-data(<Collections cnt="{$res-count}" cnt_subcolls="{$coll-count}" cnt_children="{count($children)}" root="{$collections}">{$children}</Collections>, $names[1], $names[2])
    63 
     60  let $name := cmd-model:gen-cache-id("collection", $collections, xs:string($max-depth)),
     61        $doc :=
     62    if (cmd-model:is-in-cache($name)) then
     63       cmd-model:get-from-cache($name)
     64    else
     65                let $data := cmd-model:colls($collections, $max-depth)
     66                return cmd-model:store-in-cache($name, $data)
    6467        return
    65         cmd-model:serialise-as(cmd-model:get-doc($names[1], $names[2]), $format)
     68        cmd-model:serialise-as($doc, $format)
    6669};
    6770
    … …  
    97100(: **********************
    98101        queryModel - subfunctions
    99 :)
    100 
     102C:)
    101103
    102104declare function cmd-model:elem($collections as xs:string+, $path as xs:string, $depth as xs:integer) as element() {
    … …  
    153155};
    154156
    155 (:
    156 OBSOLETE??
    157 :)
    158 declare function cmd-model:recurse-collections-model($collection as xs:string, $type-name as xs:string, $depth as xs:integer, $name as xs:string) as item()* {
    159     let $children := xdb:get-child-collections($collection)
     157
     158(: **********************
     159        getCollections - subfunctions   :)
     160
     161declare function cmd-model:colls($collections as xs:string+, $max-depth as xs:integer) as element() {
     162                let $children :=
     163                for $collection-item in $collections
     164                        return
     165                                for $collection-doc in cmd-model:get-resource-by-handle($collection-item)
     166                                return cmd-model:colls-r($collection-doc, cmd-model:get-md-collection-name($collection-doc), $collection-doc//MdSelfLink, "", $max-depth)
     167                let $res-count := sum($children/@cnt)
     168                let $coll-count := sum($children/@cnt_subcolls) + count($children)
     169                let $data := <Collections cnt="{$res-count}" cnt_subcolls="{$coll-count}" cnt_children="{count($children)}" root="{$collections}">{$children}</Collections>                     
     170                return $data
     171};
     172
     173(:
     174  Recurse down in collections.
     175:)
     176declare function cmd-model:colls-r($collection as node(), $name as xs:string, $handle as xs:string, $proxy-id as xs:string, $depth as xs:integer) as item()* {
     177  let $children :=  if ($depth eq 1) then () else cmd-model:get-children-colls($collection)
     178  (: let $dummy := util:log('debug', fn:concat(cmd-model:get-md-collection-name($collection), " ", $collection//MdSelfLink, " ", xs:string($depth), " CHILDREN = ", string-join(for $child in $children return $child//MdSelfLink, "#"))) :)
    160179    return
    161180      if (fn:exists($children)) then
    162           let $child-results :=
    163             for $child in $children
    164             return
    165               cmd-model:recurse-collections-model(fn:concat($collection, '/', xs:string($child)), $type-name, $depth, $name),
    166               $current := cmd-model:create-doc($collection, $type-name, $depth, $name)
    167           return ($current, $child-results)
     181        let $child-results :=
     182          for $child in $children
     183            (: let $child-doc := if (empty($child/unresolvable-uri)) then
     184                cmd-model:get-resource-by-handle($child/ResourceRef) else (), :)
     185            let $child-name := cmd-model:get-md-collection-name($child)
     186            let $proxyid := ($collection//ResourceProxy[ResourceRef = $child//MdSelfLink]/@id, concat("UNKNOWN proxy id:", $child//MdSelfLink))[1]
     187            return
     188              cmd-model:colls-r($child, $child-name, $child//Header/MdSelfLink, $proxyid, $depth - 1)
     189
     190          return
     191          <c n="{$name}" handle="{$handle}" proxy-id="{$proxy-id}" cnt="{sum($child-results/@cnt)}" cnt_subcolls="{if ($handle eq '') then '-1' else cmd-model:get-collection-count($handle)}" cnt_children="{count($child-results)}" >{$child-results}</c>
    168192      else
    169         cmd-model:create-doc($collection, $type-name, $depth, $name)
    170 };
    171 
    172 
    173 (: **********************
    174         getCollections - subfunctions
    175 :)
     193        <c n="{$name}" handle="{$handle}" proxy-id="{$proxy-id}" cnt_subcolls="{if ($handle eq '') then '-1' else cmd-model:get-collection-count($handle)}" cnt="{if ($handle eq '') then '-1' else cmd-model:get-resource-count($handle)}"></c>
     194
     195};
    176196
    177197(:
    … …  
    186206    util:eval(concat("$collection/ft:query(descendant::MdSelfLink, <term>", xdb:decode($id), "</term>)/ancestor::CMD"))
    187207 (: $collection/descendant::MdSelfLink[. = xdb:decode($id)]/ancestor::CMD :)
    188 
    189 };
    190 
    191 (:
    192   Recurse down in collections.
    193 :)
    194 declare function cmd-model:recurse-collections($collection as node(), $name as xs:string, $handle as xs:string, $proxy-id as xs:string, $depth as xs:integer) as item()* {
    195   let $children :=  if ($depth eq 0) then () else cmd-model:get-children-colls($collection)
    196   let $dummy := util:log('debug', fn:concat(cmd-model:get-md-collection-name($collection), " ", $collection//MdSelfLink, " ", xs:string($depth), " CHILDREN = ", string-join(for $child in $children return $child//MdSelfLink, "#")))
    197     return
    198       if (fn:exists($children)) then
    199         let $child-results :=
    200           for $child in $children
    201             (: let $child-doc := if (empty($child/unresolvable-uri)) then
    202                 cmd-model:get-resource-by-handle($child/ResourceRef) else (), :)
    203             let $child-name := cmd-model:get-md-collection-name($child)
    204             let $proxyid := ($collection//ResourceProxy[ResourceRef = $child//MdSelfLink]/@id, concat("UNKNOWN proxy id:", $child//MdSelfLink))[1]
    205             return
    206               cmd-model:recurse-collections($child, $child-name, $child//Header/MdSelfLink, $proxyid, $depth - 1)
    207 
    208           return
    209           <c n="{$name}" handle="{$handle}" proxy-id="{$proxy-id}" cnt="{sum($child-results/@cnt)}" cnt_subcolls="{if ($handle eq '') then '-1' else cmd-model:get-collection-count($handle)}" cnt_children="{count($child-results)}" >{$child-results}</c>
    210       else
    211         <c n="{$name}" handle="{$handle}" proxy-id="{$proxy-id}" cnt_subcolls="{if ($handle eq '') then '-1' else cmd-model:get-collection-count($handle)}" cnt="{if ($handle eq '') then '-1' else cmd-model:get-resource-count($handle)}"></c>
    212 
    213 };
    214 
    215 (:
    216   Get the next level collection-records (ResourceType='Metadata')
     208};
     209
     210(:  Get the next level collection-records (ResourceType='Metadata')
    217211  rely on the ResourceProxy of the parent (param)
    218212:)
    … …  
    238232};
    239233
    240 (:
    241  Try to derive a name from the collection-record (more-or-less agnostic about the actual schema.
     234(: Try to derive a name from the collection-record (more-or-less agnostic about the actual schema.
    242235:)
    243236declare function cmd-model:get-md-collection-name($collection-doc as node()) as xs:string {
    … …  
    249242:)
    250243
    251 (:
    252 
    253 :)
    254 declare function cmd-model:create-doc($collections as xs:string+, $type-name as xs:string, $depth as xs:integer, $name as xs:string) as xs:string* {
    255   (: if newer data available :)
    256     cmd-model:store-result($collections, cmd-model:elem($collections, $type-name, $depth), $name, $depth)
    257   (:else () :)
    258 };
    259 
    260 (:
    261 
    262 :)
    263 declare function cmd-model:get-result-doc($collections as xs:string+, $type-name as xs:string, $depth as xs:integer) as item()* {
    264   let $name := cmd-model:make-compound-doc-name(($collections, $type-name), "values", xs:string($depth)),
    265     $dummy := if (cmd-model:is-result-available($cmd-model:commonFreqsPath, $name)) then
    266     ()
    267     else
    268       cmd-model:create-doc($collections, $type-name, $depth, $name)
    269     return
    270       cmd-model:get-doc($cmd-model:commonFreqsPath, $name)
    271 };
    272 
    273 (:
    274  Generic get-doc(collection, docname)
    275 :)
    276 declare function cmd-model:get-doc($collection as xs:string, $doc-name as xs:string) as item()* {
    277       fn:doc(fn:concat($collection, "/", $doc-name))
    278 };
    279 
    280 
    281 (:
    282   Function for telling wether the result is already available or not.
    283 :)
    284 declare function cmd-model:is-result-available($collection as xs:string, $result-ref as xs:string) as xs:boolean {
    285   fn:doc-available(fn:concat($collection, "/", $result-ref))
    286 };
    287 
    288 (:
    289   Function for telling wether the document is available or not.
     244
     245(: Function for telling wether the document is available or not.
     246generic, currently not used
    290247:)
    291248declare function cmd-model:is-doc-available($collection as xs:string, $doc-name as xs:string) as xs:boolean {
    … …  
    293250};
    294251
    295 (:
    296   Store the calculated frequencies for reuse.
    297   If more than one collection is given the result is stored in the common
    298   collection for reuse.
    299 :)
    300 declare function cmd-model:store-result($coll-names as xs:string+, $entries as element()*, $type-name as xs:string, $depth as xs:integer) as xs:string {
     252declare function cmd-model:is-in-cache($doc-name as xs:string) as xs:boolean {
     253  fn:doc-available(fn:concat($cmd-model:cachePath, "/", $doc-name))
     254};
     255
     256declare function cmd-model:get-from-cache($doc-name as xs:string) as item()* {
     257      fn:doc(fn:concat($cmd-model:cachePath, "/", $doc-name))
     258};
     259
     260(:  Store the collection listing for given collection.
     261:)
     262declare function cmd-model:store-in-cache($doc-name as xs:string, $data as node()) as item()* {
    301263  let $clarin-writer := fn:doc("/db/clarin/writer.xml"),
    302     $dummy := xdb:login($cmd-model:cmdiDatabaseURI, $clarin-writer//write-user/text(), $clarin-writer//write-user-cred/text())
    303     return
    304         xdb:store($cmd-model:commonFreqsPath, $type-name, cmd-model:make-doc-element-of-type($type-name, $coll-names, $entries, xs:string($depth)))
    305 };
    306 
    307 (:
    308   Store the collection listing for given collection.
    309 :)
    310 declare function cmd-model:store-collection-data($data as node(), $collection-path as xs:string, $doc-name as xs:string) as xs:string? {
    311   let $clarin-writer := fn:doc("/db/clarin/writer.xml"),
    312   $dummy := xdb:login($cmd-model:cmdiDatabaseURI, $clarin-writer//write-user/text(), $clarin-writer//write-user-cred/text()),
    313   $store := (: util:catch("org.exist.xquery.XPathException", :) xdb:store($collection-path, $doc-name, $data),(: , ()) :)
    314   $stored-doc := doc(concat($collection-path, "/", $doc-name))
     264  $dummy := xdb:login($cmd-model:cachePath, $clarin-writer//write-user/text(), $clarin-writer//write-user-cred/text())
     265  let $store := (: util:catch("org.exist.xquery.XPathException", :) xdb:store($cmd-model:cachePath, $doc-name, $data), (: , ()) :)
     266  $stored-doc := fn:doc(concat($cmd-model:cachePath, "/", $doc-name))
    315267  return $stored-doc
    316268  (: moved to get-collection
    … …  
    323275
    324276(:
    325   Create document name for type () with or without collection path.
    326 :)
    327 declare function cmd-model:make-doc-name($coll-name as xs:string?, $type-name as xs:string, $depth as xs:string, $incl-path as xs:boolean) as xs:string {
    328   let $doc-name := fn:concat($type-name, $depth, $cmd-model:xmlExt)
    329   return
    330     if ($incl-path) then
    331       fn:concat($coll-name, "/", $doc-name)
    332     else
    333       $doc-name
    334 };
    335 
    336 (:
    337277  Create document name with md5-hash for selected collections (or types) for reuse.
    338278:)
    339 declare function cmd-model:make-compound-doc-name($coll-names as xs:string+, $type-name as xs:string, $depth as xs:string) as xs:string {
     279declare function cmd-model:gen-cache-id($type-name as xs:string, $keys as xs:string+, $depth as xs:string) as xs:string {
    340280  let $name-prefix := fn:concat($type-name, $depth),
    341     $sorted-names := for $coll in $coll-names order by $coll ascending return $coll
     281    $sorted-names := for $key in $keys order by $key ascending return $key
    342282    return
    343283    fn:concat($name-prefix, "-", util:hash(string-join($sorted-names, ""), "MD5"), $cmd-model:xmlExt)
    344 };
    345 
    346 (:
    347   Skapa ett element av angiven typ.
    348 :)
    349 declare function cmd-model:make-element-of-type($type-name as xs:string, $count as xs:string, $text-count as xs:string, $text-types-count as xs:string, $value as xs:string) as element() {
    350   element {$type-name} {
    351 
    352       attribute count {$freq},
    353       attribute text-count {$rank},
    354       attribute text-types-count {$text-types},
    355       text {$value}
    356   }
    357 };
    358 
    359 (:
    360   Skapa ett dokumentelement av angiven typ.
    361 :)
    362 declare function cmd-model:make-doc-element-of-type($type-name as xs:string, $coll-names as xs:string*, $entries as element()*, $depth as xs:string) as element() {
    363       let $depth-value := attribute depth {$depth},
    364       $coll-names-value := if (fn:empty($coll-names)) then () else attribute colls {fn:string-join($coll-names, ",")}
    365       return
    366         element {cmd-model:get-doc-type-element-name($type-name)} {
    367           $depth-value,
    368           $coll-names-value,
    369           attribute created {fn:current-dateTime()},
    370           $entries
    371         }
    372 };
    373 
    374 (:
    375   Get element name for document type.
    376 :)
    377 declare function cmd-model:get-doc-type-element-name($type-name as xs:string) as xs:string {
    378   $cmd-model:docTypeTerms
    379284};
    380285
    … …  
    391296};
    392297
    393 
    394 (:
    395 {cmdComponent}   //{cmdComponent}        Actor   //Actor
    396 {cmdPath}.      //{cmdPath}/{cmdComponent}      Actor.Contact.Phone     //Actor/Contact/Phone
    397 {cmdIndex} {rel} {term}         //{cmdIndex}[\. {rel} '{term}']         Actors.Actor.Sex=f      //Actors/Actor/Sex[.='f']
    398 {cmdIndex} any {term}   //{cmdIndex}[contains(. '{term}')]      Organisation.Name any University        //Organisation/Name[contains(.,'University')]
    399 and, or, and not        ?!      Organisation.Name any University and Actor.gender=m     ?!
    400 
    401 //MDGroup[Actors/Actor/Role[.='sponsor'] and Actors/Actor/Name[contains(.,'a')]]
    402 //Title[starts-with(.,'a')]
    403 //Title[starts-with(.,'A')]
    404 //Title[contains(.,'analysis')]
    405 http://demo.spraakdata.gu.se/clarin/cmd/model/stats?operation=searchRetrieve&query=//Title[contains(.,'analysis')]&collection=
    406 
    407 <record>
    408   <recordSchema>info:srw/schema/1/dc-v1.1</recordSchema>
    409   <recordPacking>xml</recordPacking>
    410   <recordData>
    411     <srw_dc:dc xmlns:srw_dc="info:srw/schema/1/dc-v1.1">
    412      <dc:title>This is a Sample Record</dc:title>
    413     </srw_dc:dc>
    414   </recordData>
    415   <recordPosition>1</recordPosition>
    416   <extraRecordData>
    417     <rel:score xmlns:rel="info:srw/extensions/2/rel-1.0">
    418       0.965
    419     </rel:rank>
    420    </extraRecordData>
    421 </record>
    422 
    423 <searchRetrieveResponse>
    424         <numberOfRecords>integer</numberOfRecords>
    425         <echoedSearchRetrieveRequest>query itself (together with the context-collection) </echoedSearchRetrieveRequest>
    426         <diagnostics>if necessary</diagnostics>
    427         <records>
    428                 ....
    429         </records>
    430 </searchRetrieveResponse>
    431 
    432 :)
Note: See TracChangeset for help on using the changeset viewer.