zorba-coders team mailing list archive

["Paul J. Lucas" <paul@xxxxxxxxxxxxx>]

Paul J. Lucas has proposed merging lp:~zorba-coders/zorba/feature-ft_module into lp:zorba.

Requested reviews:
  Paul J. Lucas (paul-lucas)
Related bugs:
  Bug #944795 in Zorba: "XQDoc doesn't handle & in URLs"
  https://bugs.launchpad.net/zorba/+bug/944795

For more details, see:
https://code.launchpad.net/~zorba-coders/zorba/feature-ft_module/+merge/113650

1. Added missing "lang" attribute to tokens generated from tokenize-nodes().
2. Added a test.
-- 
https://code.launchpad.net/~zorba-coders/zorba/feature-ft_module/+merge/113650
Your team Zorba Coders is subscribed to branch lp:zorba.

=== modified file 'src/runtime/full_text/ft_module_util.cpp'
--- src/runtime/full_text/ft_module_util.cpp	2012-06-22 03:32:00 +0000
+++ src/runtime/full_text/ft_module_util.cpp	2012-07-05 21:50:26 +0000
@@ -34,7 +34,7 @@
                                    Item const *api_item ) {
   store::Item const *const item = Unmarshaller::getInternalItem( *api_item );
   tokens_->push_back(
-    FTToken( utf8_s, utf8_len, token_no, sent_no, para_no, item )
+    FTToken( utf8_s, utf8_len, token_no, sent_no, para_no, item, lang )
   );
 }
 

=== added file 'test/rbkt/ExpQueryResults/zorba/fulltext/ft-module-tokenize-nodes-2.xml.res'
--- test/rbkt/ExpQueryResults/zorba/fulltext/ft-module-tokenize-nodes-2.xml.res	1970-01-01 00:00:00 +0000
+++ test/rbkt/ExpQueryResults/zorba/fulltext/ft-module-tokenize-nodes-2.xml.res	2012-07-05 21:50:26 +0000
@@ -0,0 +1,1 @@
+true

=== added file 'test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-nodes-2.xq'
--- test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-nodes-2.xq	1970-01-01 00:00:00 +0000
+++ test/rbkt/Queries/zorba/fulltext/ft-module-tokenize-nodes-2.xq	2012-07-05 21:50:26 +0000
@@ -0,0 +1,33 @@
+import module namespace ft = "http://www.zorba-xquery.com/modules/full-text";;
+import schema namespace fts = "http://www.zorba-xquery.com/modules/full-text";;
+
+let $x :=
+  <quotes>
+    <quote xml:lang="en">
+      I love you more than yesterday, less than tomorrow.
+    </quote>
+    <quote xml:lang="fr">
+      Je t'aime plus qu'hier, moins que demain.
+    </quote>
+  </quotes>
+
+let $includes := $x//quote
+let $excludes := ()
+
+let $tokens := ft:tokenize-nodes( $includes, $excludes )
+
+let $t1  := validate { $tokens[1] }
+let $t2  := validate { $tokens[2] }
+let $t3  := validate { $tokens[3] }
+
+let $t10 := validate { $tokens[10] }
+let $t11 := validate { $tokens[11] }
+
+return  $t1/@value = "I"        and $t1/@lang = "en"
+    and $t2/@value = "love"     and $t2/@lang = "en"
+    and $t3/@value = "you"      and $t2/@lang = "en"
+
+    and $t10/@value = "Je"      and $t10/@lang = "fr"
+    and $t11/@value = "t'aime"  and $t11/@lang = "fr"
+
+(: vim:set et sw=2 ts=2: :)