zorba-coders team mailing list archive
-
zorba-coders team
-
Mailing list archive
-
Message #20701
[Merge] lp:~zorba-coders/zorba/fix_bug_1151965 into lp:zorba
Sorin Marian Nasoi has proposed merging lp:~zorba-coders/zorba/fix_bug_1151965 into lp:zorba.
Commit message:
Implement support for DISPUTED test cases: meaning test cases that:
- are failing because we think they are wrong
- we have reported them as bugs on W3C bugzilla
Requested reviews:
Sorin Marian Nasoi (sorin.marian.nasoi)
Related bugs:
Bug #1151965 in Zorba: "Allow passing of "disputed" FOTS tests"
https://bugs.launchpad.net/zorba/+bug/1151965
Bug #1156249 in Zorba: "XSD 1.1: 0 vs -0"
https://bugs.launchpad.net/zorba/+bug/1156249
For more details, see:
https://code.launchpad.net/~zorba-coders/zorba/fix_bug_1151965/+merge/159329
Implement support for DISPUTED test cases: meaning test cases that:
- are failing because we think they are wrong
- we have reported them as bugs on W3C bugzilla
--
https://code.launchpad.net/~zorba-coders/zorba/fix_bug_1151965/+merge/159329
Your team Zorba Coders is subscribed to branch lp:zorba.
=== modified file 'test/fots/CMakeLists.txt'
--- test/fots/CMakeLists.txt 2013-04-16 20:11:02 +0000
+++ test/fots/CMakeLists.txt 2013-04-17 09:25:08 +0000
@@ -61,7 +61,7 @@
# Customized macro for FOTS tests
MACRO (expected_fots_failure)
- PARSE_ARGUMENTS (EFF "" "" "SLOW;CRASH" ${ARGN})
+ PARSE_ARGUMENTS (EFF "" "" "SLOW;CRASH;DISPUTED" ${ARGN})
LIST (LENGTH EFF_DEFAULT_ARGS _numargs)
IF (NOT _numargs EQUAL 3)
MESSAGE (FATAL_ERROR "Invalid arguments to EXPECTED_FOTS_FAILURE: ${ARGN}")
@@ -79,19 +79,29 @@
# Determine "notrun" status
SET (_notrun)
- IF (EFF_SLOW OR EFF_CRASH)
- SET (_notrun true)
- ENDIF (EFF_SLOW OR EFF_CRASH)
+ SET (_finalStatus)
+ SET (_disputed)
+ IF (EFF_SLOW)
+ SET (_notrun true)
+ SET (_finalStatus pass)
+ ENDIF (EFF_SLOW)
+ IF (EFF_CRASH)
+ SET (_notrun true)
+ SET (_finalStatus fail)
+ ENDIF (EFF_CRASH)
+ IF (EFF_DISPUTED)
+ SET (_disputed true)
+ ENDIF (EFF_DISPUTED)
# Append three values per test case - case name, bugnum, notRun
SET_PROPERTY (GLOBAL APPEND PROPERTY "_FOTS_${test_set}_CASES_"
- ${test_case} ${bugid} "${_notrun}")
+ ${test_case} ${bugid} "${_notrun}" "${_finalStatus}" "${_disputed}")
ENDMACRO (expected_fots_failure)
##
## Place all known failing FOTS tests here!
##
-#"slow" tests
+#"slow" tests. These are not run but are marked as 'pass' by FOTS driver.
EXPECTED_FOTS_FAILURE (SLOW fn-subsequence cbcl-subsequence-011 1069794)
EXPECTED_FOTS_FAILURE (SLOW fn-subsequence cbcl-subsequence-012 1069794)
EXPECTED_FOTS_FAILURE (SLOW fn-subsequence cbcl-subsequence-013 1069794)
@@ -100,11 +110,31 @@
EXPECTED_FOTS_FAILURE (SLOW app-Demos itunes 0)
EXPECTED_FOTS_FAILURE (SLOW app-Demos raytracer 0)
-#"crashing" tests
+#"crashing/hanging" tests. These are not run but marked as 'fail' by FOTS driver.
EXPECTED_FOTS_FAILURE (CRASH fn-matches.re re00975 1070533)
EXPECTED_FOTS_FAILURE (CRASH fn-matches.re re00976 1070533)
EXPECTED_FOTS_FAILURE (CRASH fn-matches.re re00976a 1070533)
-EXPECTED_FOTS_FAILURE (CRASH fn-unparsed-text-lines fn-unparsed-text-lines-052 1123835)
+
+#"disputed" tests. These test are run but marked as 'pass' by FOTS driver.
+#All these entries should have a *valid* opened bug number from W3C bugzilla.
+EXPECTED_FOTS_FAILURE (DISPUTED prod-SchemaImport qischema040 21339)
+EXPECTED_FOTS_FAILURE (DISPUTED prod-SchemaImport qischema040a 21339)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-format-date format-date-en126 21423)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-format-date format-date-en127 21423)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-format-date format-date-en128 21423)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-format-date format-date-en132 21423)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-format-date format-date-en133 21423)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-format-date format-date-en134 21423)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-matches.re re00056 21425)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-matches.re re00086 21425)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-format-integer format-integer-044 21448)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-format-date format-date-en152 21558)
+EXPECTED_FOTS_FAILURE (DISPUTED fn-format-dateTime format-dateTime-en152 21558)
+EXPECTED_FOTS_FAILURE (DISPUTED prod-AxisStep cbcl-childAxis-001 21620)
+EXPECTED_FOTS_FAILURE (DISPUTED prod-AxisStep Axes115 21620)
+EXPECTED_FOTS_FAILURE (DISPUTED prod-OrderByClause orderBy20 21619)
+EXPECTED_FOTS_FAILURE (DISPUTED prod-OrderByClause orderBy21 21619)
+
# Next three possibly unique to old RQ machine, but they need to be
# marked for the RQ to pass.
@@ -135,17 +165,8 @@
EXPECTED_FOTS_FAILURE (fn-environment-variable environment-variable-005 0)
EXPECTED_FOTS_FAILURE (fn-environment-variable environment-variable-006 0)
EXPECTED_FOTS_FAILURE (fn-environment-variable environment-variable-007 0)
-EXPECTED_FOTS_FAILURE (fn-format-date format-date-en126 0)
-EXPECTED_FOTS_FAILURE (fn-format-date format-date-en127 0)
-EXPECTED_FOTS_FAILURE (fn-format-date format-date-en128 0)
-EXPECTED_FOTS_FAILURE (fn-format-date format-date-en132 0)
-EXPECTED_FOTS_FAILURE (fn-format-date format-date-en133 0)
-EXPECTED_FOTS_FAILURE (fn-format-date format-date-en134 0)
-EXPECTED_FOTS_FAILURE (fn-format-date format-date-en152 21558)
EXPECTED_FOTS_FAILURE (fn-format-dateTime format-dateTime-006 0)
EXPECTED_FOTS_FAILURE (fn-format-dateTime format-dateTime-011 0)
-EXPECTED_FOTS_FAILURE (fn-format-dateTime format-dateTime-en152 21558)
-EXPECTED_FOTS_FAILURE (fn-format-integer format-integer-044 1162631)
EXPECTED_FOTS_FAILURE (fn-format-number numberformat41 1167427)
EXPECTED_FOTS_FAILURE (fn-format-number numberformat42 1167427)
EXPECTED_FOTS_FAILURE (fn-format-number numberformat60a 1167609)
@@ -153,9 +174,7 @@
EXPECTED_FOTS_FAILURE (fn-format-number numberformat60o 1167609)
EXPECTED_FOTS_FAILURE (fn-format-number numberformat60q 1167609)
EXPECTED_FOTS_FAILURE (fn-format-number numberformat63 1167609)
-IF (ZORBA_WITH_BIG_INTEGER)
- EXPECTED_FOTS_FAILURE (fn-format-number numberformat64 1167609)
-ENDIF (ZORBA_WITH_BIG_INTEGER)
+EXPECTED_FOTS_FAILURE (fn-format-number numberformat64 1167609)
EXPECTED_FOTS_FAILURE (fn-format-number numberformat83 1167643)
EXPECTED_FOTS_FAILURE (fn-format-number numberformat87 1167641)
EXPECTED_FOTS_FAILURE (fn-format-number numberformat88 1167641)
@@ -172,8 +191,6 @@
EXPECTED_FOTS_FAILURE (fn-iri-to-uri fn-iri-to-uri-18A 0)
EXPECTED_FOTS_FAILURE (fn-last last-24 0)
EXPECTED_FOTS_FAILURE (fn-matches cbcl-matches-038 1131304)
-EXPECTED_FOTS_FAILURE (fn-matches.re re00056 1131984)
-EXPECTED_FOTS_FAILURE (fn-matches.re re00086 1131984)
EXPECTED_FOTS_FAILURE (fn-matches.re re00288 1131985)
EXPECTED_FOTS_FAILURE (fn-matches.re re00370 1131985)
EXPECTED_FOTS_FAILURE (fn-matches.re re00480 1131985)
@@ -238,6 +255,7 @@
EXPECTED_FOTS_FAILURE (fn-unparsed-text-lines fn-unparsed-text-lines-049 0)
EXPECTED_FOTS_FAILURE (fn-unparsed-text-lines fn-unparsed-text-lines-050 0)
EXPECTED_FOTS_FAILURE (fn-unparsed-text-lines fn-unparsed-text-lines-051 0)
+EXPECTED_FOTS_FAILURE (fn-unparsed-text-lines fn-unparsed-text-lines-052 1123835)
EXPECTED_FOTS_FAILURE (fn-unparsed-text-lines fn-unparsed-text-lines-053 0)
EXPECTED_FOTS_FAILURE (fn-unparsed-text-lines fn-unparsed-text-lines-054 0)
EXPECTED_FOTS_FAILURE (math-acos math-acos-003 0)
@@ -273,8 +291,6 @@
EXPECTED_FOTS_FAILURE (prod-Annotation annotation-assertion-8 0)
EXPECTED_FOTS_FAILURE (prod-Annotation annotation-assertion-9 0)
EXPECTED_FOTS_FAILURE (prod-Annotation annotation-assertion-10 0)
-EXPECTED_FOTS_FAILURE (prod-AxisStep Axes115 0)
-EXPECTED_FOTS_FAILURE (prod-AxisStep cbcl-childAxis-001 0)
EXPECTED_FOTS_FAILURE (prod-BaseURIDecl K2-BaseURIProlog-5 0)
EXPECTED_FOTS_FAILURE (prod-CastExpr CastAs674a 0)
EXPECTED_FOTS_FAILURE (prod-CastExpr K-SeqExprCast-71b 0)
@@ -380,8 +396,6 @@
EXPECTED_FOTS_FAILURE (prod-OptionDecl.serialization Serialization-030 0)
EXPECTED_FOTS_FAILURE (prod-OptionDecl.serialization Serialization-031 0)
EXPECTED_FOTS_FAILURE (prod-OptionDecl.serialization Serialization-032 0)
-EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy20 21619)
-EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy21 21619)
EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy40 0)
EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy41 0)
EXPECTED_FOTS_FAILURE (prod-OrderByClause orderBy49 0)
@@ -416,8 +430,6 @@
EXPECTED_FOTS_FAILURE (prod-RequireProhibitFeature prohibit-all-optional-features-1 0)
EXPECTED_FOTS_FAILURE (prod-RequireProhibitFeature require-all-optional-features-7-s 0)
EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema031 0)
-EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema040 21339)
-EXPECTED_FOTS_FAILURE (prod-SchemaImport qischema040a 21339)
EXPECTED_FOTS_FAILURE (prod-SchemaImport substitution-001 0)
EXPECTED_FOTS_FAILURE (prod-SchemaImport substitution-002 0)
EXPECTED_FOTS_FAILURE (prod-SchemaImport substitution-003 0)
@@ -518,12 +530,22 @@
LIST (GET _test_cases 0 _test_case)
LIST (GET _test_cases 1 _bugnum)
LIST (GET _test_cases 2 _notrun)
- LIST (REMOVE_AT _test_cases 0 1 2)
+ LIST (GET _test_cases 3 _finalStatus)
+ LIST (GET _test_cases 4 _disputed)
+ LIST (REMOVE_AT _test_cases 0 1 2 3 4)
IF (_notrun)
SET (_notrun " notRun=\"true\"")
ENDIF (_notrun)
+ IF (_finalStatus)
+ SET (_finalStatus " finalStatus=\"${_finalStatus}\"")
+ ENDIF (_finalStatus)
+ IF (_disputed)
+ SET (_disputed " finalStatus=\"pass\"")
+ ENDIF (_disputed)
+ IF(_notrun)
FILE (APPEND "${fots_expected_failures_file}"
- " <Test name=\"${_test_case}\" bug=\"${_bugnum}\"${_notrun}/>\n")
+ " <Test name=\"${_test_case}\" bug=\"${_bugnum}\"${_notrun}${_finalStatus}${_disputed}/>\n")
+ ENDIF(_notrun)
ENDWHILE (_test_cases)
FILE (APPEND "${fots_expected_failures_file}" " </TestSet>\n")
ENDFOREACH (_test_set)
=== modified file 'test/fots_driver/cli.xq'
--- test/fots_driver/cli.xq 2013-04-09 14:22:41 +0000
+++ test/fots_driver/cli.xq 2013-04-17 09:25:08 +0000
@@ -27,7 +27,7 @@
declare namespace op = "http://www.zorba-xquery.com/options/features";;
declare namespace f = "http://www.zorba-xquery.com/features";;
-declare option op:disable "f:trace";
+(:declare option op:disable "f:trace";:)
(:~
: Path to the FOTS catalog.xml file. If the path is relative, it will be
@@ -41,7 +41,7 @@
(:~
: Path to the FOTSZorbaManifest.xml file. If the path is relative, it will be
- : resolved relative to the diractory containing this cli.xq file
+ : resolved relative to the directory containing this cli.xq file
:)
declare variable $fotsZorbaManifestPath as xs:string external :=
"FOTSZorbaManifest.xml";
@@ -58,7 +58,8 @@
(:~
: Path to the ExpectedFailures file.
:)
-declare variable $expectedFailuresPath as xs:string external :="";
+declare variable $expectedFailuresPath as xs:string external :=
+ "../../build/FOTSExpectedFailures.xml";
(:~
@@ -69,7 +70,7 @@
(:~
: A list of comma-seperated prefixes that acts as a filter for the test-sets
- : to be processed: a test-set is going to be proccesed only if its name starts
+ : to be processed: a test-set is going to be processed only if its name starts
: with a prefix in this list. An empty string means no filtering.
:
: Used by the list-test-sets, list-test-cases, and run-test-sets commands.
@@ -79,7 +80,7 @@
(:~
: A list of comma-seperated prefixes that acts as a filter for the test-cases
- : to be processed: a test-case is going to be proccesed only if its name starts
+ : to be processed: a test-case is going to be processed only if its name starts
: with a prefix in this list. An empty string means no filtering.
:
: Used by the list-test-cases, and run-test-sets commands.
@@ -88,35 +89,6 @@
(:~
- : The test cases in this list should not have their queries evaluated at all
- : (because they segfault, or hang, or take too long, etc).
- :
- : Used by the run-test-sets, run-and-report, and report commands.
- :
- : Please remember to add an equivalent for each test case in test/fots/CMakeLists.txt
- :
- : EXPECTED_FOTS_FAILURE (SLOW TEST_SET_NAME TEST_CASE_NAME BUG_NO) or
- : EXPECTED_FOTS_FAILURE (CRASH TEST_SET_NAME TEST_CASE_NAME BUG_NO)
- :)
-declare variable $exceptedTestCases as xs:string* := (
-(: ("instanceof139",
- "CastAs-UnionType-26",
- "CastAs-UnionType-30") , :) (:see bug lp:1160559 :)
- "fn-unparsed-text-lines-052", (:see bug lp:1123835 :)
- ("cbcl-subsequence-011",
- "cbcl-subsequence-012",
- "cbcl-subsequence-013",
- "cbcl-subsequence-014"), (:see bug lp:1069794. Actually passing but too slow :)
- ("re00975",
- "re00976",
- "re00976a"), (:see bug lp:1070533 :)
- "re00987", (:see bug lp:1131313 :)
- ("raytracer",
- "itunes") (: Actually passing but too slow :)
-);
-
-
-(:~
: The test sets in this list should not be not be processed at all.
:
: Used by the run-test-sets, run-and-report, and report commands.
@@ -157,7 +129,7 @@
: commands to filter the test-cases that will actually be listed/run. A test
: case qualifies if there is at least one element node under the <result>
: node of the <test-case> whose local name is equal to one of the strings
- : in the filtering set. If $assrtType is set to the empty sequence, no
+ : in the filtering set. If $assertions is set to the empty sequence, no
: filtering is done.
:
: Used in the list-test-cases and run-test-sets commands.
@@ -201,6 +173,9 @@
"'fotsPath' is set by default to the location where 'make fots-import' added the FOTS snapshot.",
"Currently this location is ZORBA_BUILD_FOLDER/test/fots/2011/QT3-test-suite/catalog.xml.",
"If you want to use other location please set 'fotsPath'.",
+ "",
+ "'expectedFailuresPath' is set by default to ${BUILDDIR}/FOTSExpectedFailures.xml.",
+ "If you want to use other location please set 'expectedFailuresPath'.",
"",
"Always try to output the result back to an XML file with nice indentation:",
"./zorba -f -q ../../test/fots_driver/cli.xq -e SET_CLI_OPTIONS_HERE -o output.xml --indent",
@@ -326,7 +301,6 @@
local:tokenize($testSetPrefixes),
$exceptedTestSets,
local:tokenize($testCasePrefixes),
- $exceptedTestCases,
$dependency,
$assertions,
xs:boolean($verbose),
@@ -349,7 +323,6 @@
$testSetName,
$exceptedTestSets,
(),
- $exceptedTestCases,
$dependency,
$assertions,
xs:boolean($verbose),
@@ -371,7 +344,6 @@
$testSetName,
$exceptedTestSets,
$testCaseName,
- $exceptedTestCases,
"",
(),
xs:boolean($verbose),
@@ -385,7 +357,7 @@
{
r:run-and-report($fotsPath,
$fotsZorbaManifestPath,
- $exceptedTestCases,
+ $expectedFailuresPath,
$exceptedTestSets)
}
=== modified file 'test/fots_driver/environment.xq'
--- test/fots_driver/environment.xq 2013-03-29 06:15:19 +0000
+++ test/fots_driver/environment.xq 2013-04-17 09:25:08 +0000
@@ -36,39 +36,7 @@
declare namespace op = "http://www.zorba-xquery.com/options/features";;
declare namespace f = "http://www.zorba-xquery.com/features";;
-declare option op:disable "f:trace";
-
-(:
-declare variable $env:hof as xs:string :=
- string-join(
- ( "declare namespace op = 'http://www.zorba-xquery.com/options/features';",
- "declare namespace f = 'http://www.zorba-xquery.com/features';",
- "declare option op:enable 'f:hof';"),
- "
");
-:)
-
-
-(:~
- : If there is a dependency on the HOF feature, return the text for enabling
- : the HOF feature within a query.
- :
- : @param $deps the dependencies of the test set and test case
- : @param $test the raw query text.
- : @return the text for enabling the HOF feature
-declare function env:enable-HOF-feature(
- $deps as element(fots:dependency)*,
- $test as xs:string
-) as xs:string?
-{
- let $check :=
- some $dep in $deps
- satisfies $dep[@type eq "feature" and contains(@value, "higherOrderFunctions")]
- return
- if ($check)
- then $env:hof
- else ()
-};
- :)
+(:declare option op:disable "f:trace";:)
(:~
@@ -111,13 +79,13 @@
(:~
- : Retruns the text for the static-base-uri declaration that must be added
+ : Returns the text for the static-base-uri declaration that must be added
: to a test-case query.
:
: NOTE: at least one of $end and $envCase is the empty sequence.
:
: @param $env the non-local environment of the test-case, if any. It is an
- : enviroment specified either at the test-set level or at the catalog
+ : environment specified either at the test-set level or at the catalog
: level and is referenced by the test-case.
: @param $envCase the local environment of the test-case, if any.
: @return the declare base-uri prolog statement.
@@ -139,7 +107,7 @@
: NOTE: at least one of $end and $envCase is the empty sequence.
:
: @param $env the non-local environment of the test-case, if any. It is an
- : enviroment specified either at the test-set level or at the catalog
+ : environment specified either at the test-set level or at the catalog
: level and is referenced by the test-case.
: @param $envCase the local environment of the test-case, if any.
: @return the text for the default element namespace declaration.
@@ -162,7 +130,7 @@
: NOTE: at least one of $end and $envCase is the empty sequence.
:
: @param $env the non-local environment of the test-case, if any. It is an
- : enviroment specified either at the test-set level or at the catalog
+ : environment specified either at the test-set level or at the catalog
: level and is referenced by the test-case.
: @param $envCase the local environment of the test-case, if any.
: @param $test the raw query text
@@ -266,7 +234,7 @@
: NOTE: at least one of $end and $envCase is the empty sequence.
:
: @param $env the non-local environment of the test-case, if any. It is an
- : enviroment specified either at the test-set level or at the catalog
+ : environment specified either at the test-set level or at the catalog
: level and is referenced by the test-case.
: @param $envCase the local environment of the test-case, if any.
: @param $envBaseURI The absolute pathname of the directory containing the
@@ -278,7 +246,7 @@
declare %ann:nondeterministic function env:add-var-decl(
$env as element(fots:environment)?,
$envCase as element(fots:environment)?,
- $envBaseURI as xs:anyURI,
+ $envBaseURI as xs:anyURI?,
$testSetBaseURI as xs:anyURI
) as xs:string?
{
@@ -503,7 +471,7 @@
:)
declare function env:set-variables(
$env as element(fots:environment)?,
- $envBaseURI as xs:anyURI
+ $envBaseURI as xs:anyURI?
) as xs:string?
{
if (empty($env))
=== modified file 'test/fots_driver/feedback.xq'
--- test/fots_driver/feedback.xq 2013-04-09 14:22:41 +0000
+++ test/fots_driver/feedback.xq 2013-04-17 09:25:08 +0000
@@ -70,7 +70,7 @@
$duration as xs:dayTimeDuration,
$verbose as xs:boolean,
$expectedFailure as xs:boolean
-) as element(fots:test-case)?
+) as element(fots:test-case)
{
if ($expectedFailure)
then feedback:pass-expected-FOTS-failure( $case,
@@ -79,7 +79,9 @@
$env,
$verbose)
else feedback:pass( $case,
- $result)
+ $result,
+ (),
+ ())
};
@@ -95,7 +97,7 @@
$zorbaQuery as xs:string,
$env as element(fots:environment)?,
$verbose as xs:boolean
-) as element(fots:test-case)?
+) as element(fots:test-case)
{
variable $info := 'Test case failed but it is marked with EXPECTED_FOTS_FAILURE in test/fots/CMakeLists.txt';
variable $status := 'pass';
@@ -117,7 +119,8 @@
as last into $case,
delete node $case/fots:description,
delete node $case/fots:created,
- delete node $case/fots:result);
+ delete node $case/fots:result,
+ delete node $case/fots:modified);
$case
}
@@ -135,14 +138,29 @@
: @param $result test result of running the test case.
: @return info about test case that passed.
:)
-declare %private function feedback:pass(
+declare %ann:sequential function feedback:pass(
$case as element(fots:test-case),
- $result as item()*
-) as element(fots:test-case)?
+ $result as item()*,
+ $comment as xs:string?,
+ $info as xs:string?
+) as element(fots:test-case)
{
if(exists($result/fots:errors))
then <fots:test-case name="{data($case/@name)}" result="wrongError" comment="{$result/fots:errors}"/>
- else <fots:test-case name="{data($case/@name)}" result="pass"/>
+ else
+ {
+ variable $ret := <fots:test-case name="{data($case/@name)}"
+ result="pass"/>;
+
+ (if(exists($comment))
+ then insert node attribute comment{$comment} as last into $ret
+ else (),
+ if(exists($info))
+ then insert node attribute info{$info} as last into $ret
+ else ());
+
+ $ret
+ }
};
@@ -160,13 +178,14 @@
$env as element(fots:environment)?,
$duration as xs:dayTimeDuration,
$verbose as xs:boolean,
- $expectedFailure as xs:boolean
-) as element(fots:test-case)?
+ $expectedFailure as xs:boolean,
+ $comment as xs:string?,
+ $info as xs:string?
+) as element(fots:test-case)
{
trace($testSetName, "test set name");
trace("above test case failed", "result");
- variable $info := 'Test case passed but it is marked with EXPECTED_FOTS_FAILURE in test/fots/CMakeLists.txt';
variable $status := 'fail';
if ($verbose)
@@ -189,18 +208,26 @@
as last into $case,
delete node $case/fots:description,
delete node $case/fots:created,
- delete node $case/fots:result);
+ delete node $case/fots:result,
+ delete node $case/fots:modified);
$case
}
}
- else if ($expectedFailure)
- then <fots:test-case name="{data($case/@name)}"
- result="{$status}"
- comment="{$info}"/>
+ else
+ {
+ variable $ret := <fots:test-case name="{data($case/@name)}"
+ result="{$status}"/>;
- else <fots:test-case name="{data($case/@name)}"
- result="{$status}"/>
+ (if(exists($comment))
+ then insert node attribute comment{$comment} as last into $ret
+ else (),
+ if(exists($info))
+ then insert node attribute info{$info} as last into $ret
+ else ());
+
+ $ret
+ }
};
@@ -215,7 +242,7 @@
declare function feedback:not-run(
$case as element(fots:test-case),
$error as xs:string?
-) as element(fots:test-case)?
+) as element(fots:test-case)
{
trace(data($case/@name), "processing test case :");
trace("Above test case was not run.","");
@@ -236,7 +263,7 @@
declare function feedback:not-applicable(
$case as element(fots:test-case),
$dependencyError as xs:string
-) as element(fots:test-case)?
+) as element(fots:test-case)
{
trace(data($case/@name), "processing test case :");
trace($dependencyError, "Dependency error :");
=== modified file 'test/fots_driver/fots-driver.xq'
--- test/fots_driver/fots-driver.xq 2013-04-15 12:07:41 +0000
+++ test/fots_driver/fots-driver.xq 2013-04-17 09:25:08 +0000
@@ -56,21 +56,17 @@
declare namespace ann =
"http://www.zorba-xquery.com/annotations";;
-declare namespace op = "http://www.zorba-xquery.com/options/features";;
-declare namespace f = "http://www.zorba-xquery.com/features";;
-(:declare option op:disable "f:trace";:)
-
(:~
- : Returns the names of all qualifying test sets.
+ : Returns the names of all qualifying test-sets.
:
- : A test set qualifies if its name starts with one of the prefixes listed in
- : $testSetPrefixes. If $testSetPrefixes is the empty sequence, then all test
- : sets qualify.
+ : A test-set qualifies if its name starts with one of the prefixes listed in
+ : $testSetPrefixes. If $testSetPrefixes is the empty sequence, then all test-sets
+ : qualify.
:
: @param $fotsPath path to the FOTS catalog file.
- : @param $testSetPrefixes name criteria for the test sets
+ : @param $testSetPrefixes name criteria for the test-sets
: (empty sequence means all).
- : @return names of qualifying FOTS test sets.
+ : @return names of qualifying FOTS test-sets.
:)
declare %ann:nondeterministic function driver:list-test-sets(
$fotsPath as xs:string,
@@ -91,9 +87,9 @@
(:~
- : Returns the names of all qualifying test cases.
+ : Returns the names of all qualifying test-cases.
:
- : A test case qualifies if (a) it belongs to a qualifing test-set, and (b)
+ : A test-case qualifies if (a) it belongs to a qualifying test-set, and (b)
: its applicable dependencies include a user-provided dependency, or no
: dependency was provided by the user, and (c) its expected-result assertions
: include at least one of the assertions in a user-provided set of assertions,
@@ -115,13 +111,13 @@
: filtering set.
:
: @param $fotsPath path to the FOTS catalog file.
- : @param $testSetNames name criteria for the test sets
+ : @param $testSetNames name criteria for the test-sets
: (empty sequence means all test sets).
: @param $dependency dependency used to filter test-cases.
: (empty string means no filtering).
: @param $assert set of expected-result assertions used to filter test-cases.
: (empty sequence means no filtering).
- : @return names of qualifying FOTS test cases.
+ : @return names of qualifying FOTS test-cases.
:)
declare %ann:nondeterministic function driver:list-test-cases(
$fotsPath as xs:string,
@@ -147,19 +143,19 @@
(:~
- : Helper function. Returns the names of all qualifying test cases within a
- : given test set. A test case qualifies if (a) its applicable dependencies
+ : Helper function. Returns the names of all qualifying test-cases within a
+ : given test-set. A test-case qualifies if (a) its applicable dependencies
: include a user-provided dependency, or no dependency was provided by the
: user, and (b) its expected-result assertions include at least one of the
: assertions in a user-provided set of assertions, or no expected-result
: assertions were provided by the user.
:
- : @param $testSetDoc root node of the xml document the specifies the test set.
+ : @param $testSetDoc root node of the xml document the specifies the test-set.
: @param $dependency dependency used to filter test-cases.
: (empty string means no filtering).
: @param $assert set of expected-result assertions used to filter test-cases.
: (empty sequence means no filtering).
- : @return names of qualifying FOTS test cases.
+ : @return names of qualifying FOTS test-cases.
:)
declare %private function driver:list-test-cases(
$testSetDoc as document-node(),
@@ -197,7 +193,7 @@
(:~
: Helper function to check whether a user-specified dependency (given as an
: encoded string) "matches" at least one of the dependencies in a given list
- : of dependecies (given as a sequence of <dependency> elements)
+ : of dependencies (given as a sequence of <dependency> elements)
:)
declare %private function driver:matches-dependency(
$dependencies as element(fots:dependency)*,
@@ -225,7 +221,7 @@
(:~
: Helper function to return the local names of all the nodes under a <result>
- : child of a given <test-case>.
+ : child of a given test-case.
:)
declare %private function driver:list-assertions(
$case as element(fots:test-case)
@@ -237,16 +233,16 @@
(:~
- : For each qualifying test case, this functions returns a string containing
- : the test-case name and the full filepath of its associated test-set file.
+ : For each qualifying test-case, this functions returns a string containing
+ : the test-case name and the full file path of its associated test-set file.
:
- : A test case qualifies if its <test> node (which contains the query text)
+ : A test-case qualifies if its <test> node (which contains the query text)
: matches the given pattern using the given flags.
:
: @param $fotsPath path to the FOTS catalog file.
: @param $pattern pattern.
: @param $flags flags.
- : @return names of qualifying FOTS test case and the filepaths of their
+ : @return names of qualifying test-case and the file path of their
: containing test-set files.
:)
declare %ann:nondeterministic function driver:list-matching-test-cases(
@@ -277,16 +273,7 @@
: driver:run() for more info.
:
: Process a specified subset of all test-cases and report the outcome for
- : each such test case.
- :
- : Processing a test-case results in one of the following outcomes:
- : 1. Pass: the query was evaluated and its result matched the expected result.
- : 2. Fail: the query was evaluated and its result did not match the expected
- : result.
- : 3. Non-applicable: the query was not evaluated because at least one of its
- : applicable dependencies was violated.
- : 4. Not-run: the query was not evaluated because the name of the test-case
- : appears in the $exceptedTestCases parameter.
+ : each such test-case.
:
: The subset of test-cases to process is specified via the $exceptedTestSets,
: $testSetPrefixes, $testCasePrefixes, $dependency, and assertions parameters.
@@ -311,14 +298,13 @@
: @param $exceptedTestSets names of test sets that should NOT be precessed.
: @param $testCasePrefixes prefixes to filter the test cases to process
: (empty seq means no filtering).
- : @param $exceptedTestCases names of test cases that should NOT be run.
: @param $dependency dependency used to filter test-cases.
: (empty string means no filtering).
: @param $assertions set of expected-result assertions used to filter test-cases.
: (empty sequence means no filtering).
: @param $verbose if true, the resulting XML tree will contain more details
: about each processed test-case.
- : @param $expectedFailures the root node of the ExpectedFailures.xml file.
+ : @param $expectedFailuresPath the path to the FOTSExpectedFailures.xml.
: @param $cliMode the cli command.
: @param $usePlanSerializer if true the plan serializer is used.
: @return an XML tree containing info about all the processed tests-cases
@@ -329,7 +315,6 @@
$testSetPrefixes as xs:string*,
$exceptedTestSets as xs:string*,
$testCasePrefixes as xs:string*,
- $exceptedTestCases as xs:string*,
$dependency as xs:string,
$assertions as xs:string*,
$verbose as xs:boolean,
@@ -338,16 +323,18 @@
$usePlanSerializer as xs:boolean
) as element(fots:test-cases)
{
- trace($fotsPath, "Path to FOTS catalog.xml was set to: ");
- trace($zorbaManifestPath, "Path to FOTSZorbaManifest set to :");
- trace($expectedFailuresPath, "Path to ExpectedFailures.xml set to:");
+ trace($fotsPath, "Path to FOTS catalog.xml set to:");
+ trace($expectedFailuresPath, "Path to FOTSExpectedFailures.xml set to:");
+ trace($zorbaManifestPath, "Path to FOTSZorbaManifest set to :");
trace($usePlanSerializer, "'usePlanSerializer' set to:");
try
{
let $FOTSCatalog := doc(resolve-uri($fotsPath))
-
let $zorbaManifest := doc(resolve-uri($zorbaManifestPath))
+ let $expectedFailures := if($expectedFailuresPath ne '')
+ then doc(resolve-uri($expectedFailuresPath))
+ else ()
let $testSetNames :=
if(($cliMode = 'run-test-case') ||
@@ -406,36 +393,20 @@
}
}
- let $expectedFailures :=
- {
- if ($expectedFailuresPath eq '')
- then ()
- else doc($expectedFailuresPath)
- }
-
- return driver:run-fots($FOTSCatalog,
- resolve-uri(util:parent-folder($fotsPath)),
- $zorbaManifest,
- $testSetNames,
- $testCaseNames,
- $exceptedTestCases,
- $verbose,
- $expectedFailures,
- $cliMode,
- $usePlanSerializer)
+ return driver:check-FOTS($FOTSCatalog,
+ resolve-uri(util:parent-folder($fotsPath)),
+ $zorbaManifest,
+ $testSetNames,
+ $testCaseNames,
+ $expectedFailures,
+ $verbose,
+ $cliMode,
+ $usePlanSerializer)
}
catch err:FODC0002
{
error($err:code,
- concat("
Please make sure the passed 'fotsPath' points to the",
- " exact location of the FOTS catalog.xml:
",
- resolve-uri($fotsPath),
- "
 that the passed 'fotsZorbaManifestPath' points to",
- " a file in the same folder as cli.xq:
",
- resolve-uri($zorbaManifestPath),
- "
 that the passed 'expectedFailuresPath' points to",
- " the ExpectedFailures.xml file:
",
- $expectedFailuresPath))
+ $err:description)
}
};
@@ -474,14 +445,13 @@
: @param $exceptedTestSets names of test sets that should NOT be precessed.
: @param $testCasePrefixes prefixes to filter the test cases to process
: (empty seq means no filtering).
- : @param $exceptedTestCases names of test cases that should NOT be run.
: @param $dependency dependency used to filter test-cases.
: (empty string means no filtering).
: @param $assertions set of expected-result assertions used to filter test-cases.
: (empty sequence means no filtering).
: @param $verbose if true, the resulting XML tree will contain more details
: about each processed test-case.
- : @param $expectedFailures the root node of the ExpectedFailures.xml file.
+ : @param $expectedFailuresPath the path to the FOTSExpectedFailures.xml.
: @param $cliMode the cli command.
: @param $usePlanSerializer if true the plan serializer is used.
: @return an XML tree containing info about all the processed tests-cases
@@ -492,7 +462,6 @@
$testSetName as xs:string*,
$exceptedTestSets as xs:string*,
$testCasePrefixes as xs:string*,
- $exceptedTestCases as xs:string*,
$dependency as xs:string,
$assertions as xs:string*,
$verbose as xs:boolean,
@@ -501,17 +470,17 @@
$usePlanSerializer as xs:boolean
) as element(fots:test-cases)
{
- trace($fotsPath, "Path to FOTS catalog.xml was set to: ");
+ trace($fotsPath, "Path to FOTS catalog.xml set to:");
+ trace($expectedFailuresPath, "Path to FOTSExpectedFailures.xml set to:");
trace($zorbaManifestPath, "Path to FOTSZorbaManifest set to :");
- trace($expectedFailuresPath, "Path to ExpectedFailures.xml set to:");
trace($usePlanSerializer, "'usePlanSerializer' set to:");
try
{
let $FOTSCatalog := doc(resolve-uri($fotsPath))
-
let $zorbaManifest := doc(resolve-uri($zorbaManifestPath))
-
+ let $expectedFailures := doc(resolve-uri($expectedFailuresPath))
+
let $testSetNames :=
if(empty($testSetName))
then
@@ -560,52 +529,27 @@
}
}
- let $expectedFailures :=
- {
- if ($expectedFailuresPath eq '')
- then ()
- else doc($expectedFailuresPath)
- }
-
- return driver:run-fots($FOTSCatalog,
+ return driver:check-FOTS($FOTSCatalog,
resolve-uri(util:parent-folder($fotsPath)),
$zorbaManifest,
$testSetNames,
$testCaseNames,
- $exceptedTestCases,
+ $expectedFailures,
$verbose,
- $expectedFailures,
$cliMode,
$usePlanSerializer)
}
catch err:FODC0002
{
error($err:code,
- concat("
Please make sure the passed 'fotsPath' points to the",
- " exact location of the FOTS catalog.xml:
",
- resolve-uri($fotsPath),
- "
 that the passed 'fotsZorbaManifestPath' points to",
- " a file in the same folder as cli.xq:
",
- resolve-uri($zorbaManifestPath),
- "
 that the passed 'expectedFailuresPath' points to",
- " the ExpectedFailures.xml file:
",
- $expectedFailuresPath))
+ $err:description)
}
};
(:~
- : A helper functrion to process a specified subset of all test-cases and
- : report the outcome for each such test case.
- :
- : Processing a test-case results in one of the following outcomes:
- : 1. Pass: the query was evaluated and its result matched the expected result.
- : 2. Fail: the query was evaluated and its result did not match the expected
- : result.
- : 3. Non-applicable: the query was not evaluated because at least one of its
- : applicable dependencies was violated.
- : 4. Not-run: the query was not evaluated because the name of the test-case
- : appears in the $exceptedTestCases parameter.
+ : A helper function to process a specified subset of all test-cases and
+ : report the outcome for each such test-case.
:
: The subset of test-cases to process is specified via the $testSetNames
: and $testCaseNames parameters. A test-case will be processed only if it
@@ -615,195 +559,271 @@
: (b) $testCaseNames is the empty sequence, or the name of the test-case
: appears in $testCaseNames.
:
- : @param $FOTSCatalog the root node of the FOTS catalog doc.
- : @param $catalogBaseURI the URI of the directory containing the catalog.xml file
+ : @param $FOTSCatalog the root node of the FOTS catalog document.
+ : @param $catalogBaseURI the URI of the directory containing the FOTS catalog file.
: @param $FOTSZorbaManifest the root node of the doc that describes optional
: features and implementation-defined items in Zorba.
- : @param $testSetNames names of the test sets to run (empty seq means all)
- : @param $testCaseNames names of the test sets to run (empty seq means all)
- : @param $exceptedTestCases names of test cases that should NOT be run
- : @param $exceptedTestSets names of test sets that should NOT be run
+ : @param $testSetNames names of the test-sets to run (empty seq means all)
+ : @param $testCaseNames names of the test-cases to run (empty seq means all)
+ : @param $expectedFailures the root node of the ExpectedFailures.xml file.
: @param $verbose if true, the resulting XML tree will contain more details
: about each processed test-case.
- : @param $expectedFailures the root node of the ExpectedFailures.xml file.
: @param $cliMode the cli command.
: @param $usePlanSerializer if true the plan serializer is used.
- : @return an XML tree containing info about all the processed tests-cases
+ : @return an XML tree containing info about all the processed test-cases.
:)
-declare %private %ann:sequential function driver:run-fots(
+declare %private %ann:sequential function driver:check-FOTS(
$FOTSCatalog as document-node(),
$catalogBaseURI as xs:anyURI,
$FOTSZorbaManifest as document-node(),
$testSetNames as xs:string*,
$testCaseNames as xs:string*,
- $exceptedTestCases as xs:string*,
+ $expectedFailures as document-node()?,
$verbose as xs:boolean,
- $expectedFailures as document-node()?,
$cliMode as xs:string,
$usePlanSerializer as xs:boolean
) as element(fots:test-cases)
{
- <fots:test-cases>
- {
- let $testSets := if (empty($testSetNames))
- then $FOTSCatalog//fots:test-set
- else $FOTSCatalog//fots:test-set[@name = $testSetNames]
-
- for $testSet in $testSets
-
- let $testSetName := $testSet/@name
-
- let $testSetURI := resolve-uri($testSet/@file, $catalogBaseURI)
-
- let $testSetDoc := doc($testSetURI)
-
- let $depMet :=
- env:check-dependencies($testSetDoc/fots:test-set/fots:dependency,
- $FOTSZorbaManifest)
-
- return
- if (exists($depMet)) then
- {
- <fots:test-set name="{$testSetName}">
- {
- for $testCase in $testSetDoc/fots:test-set/fots:test-case
-
- let $envName := data($testCase/fots:environment/@ref)
-
- let $envTestSet := $testSetDoc/fots:test-set/fots:environment[@name eq $envName]
-
- let $envCatalog := $FOTSCatalog/fots:catalog/fots:environment[@name eq $envName]
-
- let $isExcepted := $exceptedTestCases[. eq $testCase/@name]
-
- where empty($testCaseNames) or $testCaseNames[. eq $testCase/@name]
-
- return
- if ($isExcepted)
- then
- feedback:not-run($testCase, ())
- else
- feedback:not-applicable($testCase,
- string-join($depMet,''))
- }
- </fots:test-set>
- }
- else
- {
- <fots:test-set name="{$testSetName}">
- {
- for $testCase in $testSetDoc/fots:test-set/fots:test-case
-
- let $envName := data($testCase/fots:environment/@ref)
-
- let $envTestSet := $testSetDoc/fots:test-set/fots:environment[@name eq $envName]
-
- let $envCatalog := $FOTSCatalog/fots:catalog/fots:environment[@name eq $envName]
-
- let $isExcepted := $exceptedTestCases[. eq $testCase/@name]
-
- let $depMet := env:check-dependencies($testCase/fots:dependency,
- $FOTSZorbaManifest)
- (: the following 3 test sets contain some test cases that need DTD validation for the document bound as context item :)
- let $mayNeedDTDValidation := exists(index-of(('fn-id', 'fn-idref', 'app-FunctxFunctx'), $testSetName))
-
- where empty($testCaseNames) or $testCaseNames[. eq $testCase/@name]
-
- return
- {
- if ($isExcepted) then
- {
- if (exists($expectedFailures) and
- empty($expectedFailures/failures/TestSet[@name eq $testSetName]/Test[@name eq $testCase/@name]))
- then
- feedback:fail($testCase,
- (),
- '',
- $testSetName,
- (),
- xs:dayTimeDuration("PT0S"),
- $verbose,
- fn:false())
- else
- feedback:not-run($testCase,())
- }
- else if (exists($depMet)) then
- {
- feedback:not-applicable($testCase,
- string-join($depMet, ''))
- }
- else if (empty($envTestSet)) then
- {
- driver:test($testCase,
- $envCatalog,
+ <fots:test-cases>{
+ let $testSets := if (empty($testSetNames))
+ then $FOTSCatalog//fots:test-set
+ else $FOTSCatalog//fots:test-set[@name = $testSetNames]
+
+ for $testSet in $testSets
+ let $testSetName := $testSet/@name
+ let $testSetURI := resolve-uri($testSet/@file, $catalogBaseURI)
+ let $tsDoc := doc($testSetURI)
+ return
+ driver:check-test-set($FOTSCatalog/fots:catalog/fots:environment,
$catalogBaseURI,
- ($testCase/fots:dependency,
- $testSetDoc/fots:test-set/fots:dependency),
- $testSetName,
- $testSetURI,
- $verbose,
- $expectedFailures//TestSet[@name eq $testSetName]/Test[@name eq $testCase/@name],
- $cliMode,
- $mayNeedDTDValidation,
- $usePlanSerializer)
- }
- else
- {
- driver:test($testCase,
- $envTestSet,
- $testSetURI,
- ($testCase/fots:dependency,
- $testSetDoc/fots:test-set/fots:dependency),
- $testSetName,
- $testSetURI,
- $verbose,
- $expectedFailures//TestSet[@name eq $testSetName]/Test[@name eq $testCase/@name],
- $cliMode,
- $mayNeedDTDValidation,
- $usePlanSerializer)
- }
- }
- }
- </fots:test-set>
- }
+ $FOTSZorbaManifest,
+ $tsDoc,
+ $testSetURI,
+ $expectedFailures/failures/TestSet[@name eq $testSetName],
+ $testCaseNames,
+ $verbose,
+ $cliMode,
+ $usePlanSerializer)
+ }</fots:test-cases>
+};
+
+
+(:~
+ : Checks a single test-set.
+ :
+ : @param $catalogEnvironments all the 'environments' defined in the FOTS catalog.
+ : @param $catalogBaseURI the URI of the directory that contains the FOTS catalog.
+ : @param $FOTSZorbaManifest the root node of the doc that describes optional
+ : features and implementation-defined items in Zorba.
+ : @param $tsDoc the root node of the test-set document.
+ : @param $tsBaseURI the URI of the directory that contains the file of the
+ : associated test-set.
+ : @param $expFailuresTS the 'TestSet' element from FOTSExpectedFailures.xml.
+ : @param $testCaseNames names of the test-cases to run (empty seq means all).
+ : @param $verbose if true, the resulting XML tree will contain more details
+ : about the test-case.
+ : @param $cliMode the cli command.
+ : @param $usePlanSerializer if true the plan serializer is used.
+ : @return an XML tree containing the result of checking/running the test-cases.
+ :)
+declare %private %ann:sequential function driver:check-test-set(
+ $catalogEnvironments as element(fots:environment)*,
+ $catalogBaseURI as xs:anyURI,
+ $FOTSZorbaManifest as document-node(),
+ $tsDoc as document-node(),
+ $tsBaseURI as xs:anyURI,
+ $expFailuresTS as element(TestSet)?,
+ $testCaseNames as xs:string*,
+ $verbose as xs:boolean,
+ $cliMode as xs:string,
+ $usePlanSerializer as xs:boolean
+) as element(fots:test-set)
+{
+ let $testSetEnvironments := $tsDoc/fots:test-set//fots:environment/@name
+ let $tsDepMet := env:check-dependencies($tsDoc/fots:test-set/fots:dependency,
+ $FOTSZorbaManifest)
+ let $tsName := $tsDoc/fots:test-set/@name
+ return
+ <fots:test-set name="{$tsName}">{
+ for $testCase in $tsDoc/fots:test-set/fots:test-case
+ where empty($testCaseNames) or $testCaseNames[. eq $testCase/@name]
+ return
+ driver:check-test-case($expFailuresTS/Test[@name = $testCase/@name],
+
+ $testCase,
+
+ $tsName,
+
+ $tsBaseURI,
+
+ $tsDepMet,
+
+ $tsDoc/fots:test-set/fots:dependency,
+
+ $FOTSZorbaManifest,
+
+ if(empty($testCase/fots:environment/@ref))
+ then ()
+ else if (exists($catalogEnvironments/following-sibling::*[@name eq $testCase/fots:environment/@ref]))
+ then $catalogEnvironments/following-sibling::*[@name eq $testCase/fots:environment/@ref]
+ else $tsDoc/fots:test-set/fots:environment[@name eq $testCase/fots:environment/@ref],
+
+ if(empty($testCase/fots:environment/@ref))
+ then ()
+ else if (exists($catalogEnvironments/following-sibling::*[@name eq $testCase/fots:environment/@ref]))
+ then $catalogBaseURI
+ else $tsBaseURI,
+
+ exists(index-of(('fn-id', 'fn-idref', 'app-FunctxFunctx'), $tsName)),
+
+ $verbose,
+
+ $cliMode,
+
+ $usePlanSerializer)
}
- </fots:test-cases>
+ </fots:test-set>
};
-
(:~
- : Runs a single test case.
+ : Checks a single test-case.
:
- : @param $case the test case to run.
- : @param $env the non-local environment to use, if any. It is an enviroment
+ : @param $expFailuresTC the 'Test' element from FOTSExpectedFailures.xml
+ : for the parent test-set.
+ : @param $testCase the test-case to run.
+ : @param $tsName the name of the parent test-set.
+ : @param $tsBaseURI the URI of the directory that contains the file of the
+ : associated test-set.
+ : @param $deps the result of checking the test-set level dependencies.
+ : @param $tsDependencies the test-set level dependencies.
+ : @param $FOTSZorbaManifest the root node of the doc that describes optional
+ : features and implementation-defined items in Zorba.
+ : @param $env the non-local environment to use, if any. It is an environment
: specified either at the test-set level or at the catalog level and
: is referenced by the test-case.
: @param $envBaseURI the URI of the directory containing the file where the
- : envoronment is specified in. It is used to calculate the full URI
+ : environment is specified in. It is used to calculate the full URI
: for the different children of the <environment> that have a @file
: attribute.
- : @param $deps the dependencies that should be checked for given test case.
- : @param $testSetName the name of the environment test set.
- : @param $testSetBaseURI the URI of the directory that contains the file of the
- associated test set.
- : @param $verbose if true, the resulting XML tree will contain more details
- : about the test-case.
- : @param $expectedFailure the Test element from the ExpectedFailures.xml file.
- : @param $cliMode the cli command.
: @param $mayNeedDTDValidation true if the test case may need DTD validation
: for the document bound as context item.
- : @param $usePlanSerializer if true the plan serializer is used.
- : @return an XML tree containing info about the result of running the test case.
- :)
-declare %ann:sequential function driver:test(
+ : @param $verbose if true, the resulting XML tree will contain more details
+ : about the test-case.
+ : @param $cliMode the cli command.
+ : @param $usePlanSerializer if true the plan serializer is used.
+ : @return an XML tree containing the result of checking/running the test-case.
+ :)
+declare %ann:sequential function driver:check-test-case(
+ $expFailureTC as element(Test)?,
+ $testCase as element(fots:test-case),
+ $tsName as xs:string,
+ $tsBaseURI as xs:anyURI,
+ $tsDepMet as xs:string*,
+ $tsDependencies as element(fots:dependency)*,
+ $FOTSZorbaManifest as document-node(),
+ $env as element(fots:environment)?,
+ $envBaseURI as xs:anyURI?,
+ $mayNeedDTDValidation as xs:boolean,
+ $verbose as xs:boolean,
+ $cliMode as xs:string,
+ $usePlanSerializer as xs:boolean
+) as element(fots:test-case)?
+{
+ let $tcDepMet := env:check-dependencies($testCase/fots:dependency,
+ $FOTSZorbaManifest)
+ return
+ if (exists($expFailureTC[@notRun="true"]))
+ then
+ {
+(:
+ concat("Test case was not run because it is marked as EXPECTED_FOTS_FAILURE ",
+ $status,
+ " in test/fots/CMakeLists.txt.",
+ " For details please check out https://bugs.launchpad.net/zorba/+bug/";,
+ $bugNo)
+:)
+
+ if($expFailureTC/@finalStatus = "pass")
+ then
+ feedback:pass($testCase,
+ (),
+ (),
+ concat("Test case was not run because it is marked as",
+ " EXPECTED_FOTS_FAILURE SLOW in test/fots/CMakeLists.txt.",
+ "For details please check out https://bugs.launchpad.net/zorba/+bug/";,
+ $expFailureTC/@bug))
+ else
+ feedback:fail($testCase,
+ (),
+ '',
+ $tsName,
+ (),
+ xs:dayTimeDuration("PT0S"),
+ $verbose,
+ fn:false(),
+ (),
+ concat("Test case was not run because it is marked as",
+ " EXPECTED_FOTS_FAILURE CRASH in test/fots/CMakeLists.txt.",
+ "For details please check out https://bugs.launchpad.net/zorba/+bug/";,
+ $expFailureTC/@bug))
+ }
+ else if (exists($tsDepMet) or exists($tcDepMet))
+ then
+ {
+ feedback:not-applicable($testCase,
+ string-join(($tsDepMet, $tcDepMet), ''))
+ }
+ else
+ {
+ driver:run-test-case($testCase,
+ $env,
+ $envBaseURI,
+ ($testCase/fots:dependency,
+ $tsDependencies),
+ $tsName,
+ $tsBaseURI,
+ $verbose,
+ $expFailureTC,
+ $cliMode,
+ $mayNeedDTDValidation,
+ $usePlanSerializer)
+ }
+};
+
+(:~
+ : Runs a single test-case.
+ :
+ : @param $case the test-case to run.
+ : @param $env the non-local environment to use, if any. It is an environment
+ : specified either at the test-set level or at the catalog level and
+ : is referenced by the test-case.
+ : @param $envBaseURI the URI of the directory containing the file where the
+ : environment is specified in. It is used to calculate the full URI
+ : for the different children of the <environment> that have a @file
+ : attribute.
+ : @param $deps the dependencies that should be checked for given test-case.
+ : These may be defined at test-set level and/or test-case level.
+ : @param $testSetName the name of the test-set.
+ : @param $testSetBaseURI the URI of the directory that contains the file of the
+ associated test-set.
+ : @param $verbose if true, the resulting XML tree will contain more details
+ : about the test-case.
+ : @param $expFailureTC the 'Test' element from the ExpectedFailures.xml file.
+ : @param $cliMode the cli command.
+ : @param $mayNeedDTDValidation true if the test-case may need DTD validation
+ : for the document bound as context item.
+ : @param $usePlanSerializer if true the plan serializer is used.
+ : @return an XML tree containing the result of running the test-case.
+ :)
+declare %private %ann:sequential function driver:run-test-case(
$case as element(fots:test-case),
$env as element(fots:environment)?,
$envBaseURI as xs:anyURI?,
$deps as element(fots:dependency)*,
- $testSetName as xs:string?,
+ $testSetName as xs:string,
$testSetBaseURI as xs:anyURI,
$verbose as xs:boolean,
- $expectedFailure as element(Test)?,
+ $expFailureTC as element(Test)?,
$cliMode as xs:string,
$mayNeedDTDValidation as xs:boolean,
$usePlanSerializer as xs:boolean
@@ -836,8 +856,6 @@
env:decl-namespaces($env, $envCase, $test),
- (:env:enable-HOF-feature(($deps, $case//fots:dependency), $test), :)
-
env:decl-decimal-formats(($env/fots:decimal-format,
$envCase/fots:decimal-format)),
@@ -885,7 +903,7 @@
variable $prerequisitesError as xs:string? := env:check-prerequisites($case, $env);
- if (feedback:check-pass($result, $queryName, $testSetName, $expectedFailure))
+ if (feedback:check-pass($result, $queryName, $testSetName, $expFailureTC))
then
feedback:pass($case,
$result,
@@ -893,7 +911,7 @@
$env,
$duration,
$verbose,
- exists($expectedFailure))
+ exists($expFailureTC))
(:
If the test case did not pass, we check to see if the failure is caused
by a environment that requires setting of a COLLATION or COLLECTION.
@@ -912,7 +930,9 @@
$env,
$duration,
$verbose,
- exists($expectedFailure))
+ exists($expFailureTC),
+ (),
+ ())
}
}
catch *
@@ -928,7 +948,9 @@
$env,
xs:dayTimeDuration("PT0S"),
$verbose,
- exists($expectedFailure))
+ exists($expFailureTC),
+ (),
+ ())
}
};
@@ -1043,4 +1065,3 @@
"
"
)
};
-
=== modified file 'test/fots_driver/reporting.xq'
--- test/fots_driver/reporting.xq 2013-04-09 14:22:41 +0000
+++ test/fots_driver/reporting.xq 2013-04-17 09:25:08 +0000
@@ -55,8 +55,7 @@
: Loops through the test-sets, executes them and reports results.
: @param $FOTSCatalogFilePath path to the FOTS catalog file.
: @param $FOTSZorbaManifestPath path to the FOTS Zorba manifest file.
- : @param $exceptedTestCases lists of test cases that are not run( empty
- : sequence means all test cases will be run).
+ : @param $expectedFailuresPath the path to the FOTSExpectedFailures.xml.
: @param $exceptedTestSets lists of test sets that are not run(empty sequence
: means all test sets will be run).
: @return a report of tests that were executed.
@@ -64,52 +63,29 @@
declare %ann:sequential function reporting:run-and-report(
$FOTSCatalogFilePath as xs:string,
$FOTSZorbaManifestPath as xs:string,
- $exceptedTestCases as xs:string*,
+ $expectedFailuresPath as xs:string,
$exceptedTestSets as xs:string*
) as document-node()
{
- try
- {
- {
- variable $FOTSCatalog := doc(trace(resolve-uri($FOTSCatalogFilePath),
- "Path to FOTS catalog.xml set to: "));
-
- variable $catalogBaseURI := resolve-uri(util:parent-folder($FOTSCatalogFilePath));
-
- variable $FOTSZorbaManifest := doc(trace(resolve-uri($FOTSZorbaManifestPath),
- "Path to FOTSZorbaManifest set to:"));
-
- variable $results := driver:run-fots($FOTSCatalogFilePath,
- $FOTSZorbaManifestPath,
- (),
- $exceptedTestSets,
- (),
- $exceptedTestCases,
- '',
- (),
- fn:false(),
- '',
- 'run-test-sets',
- fn:false());
-
- file:write("results.xml",
- $results,
- $util:writeXML);
-
- reporting:W3C-reporting($results,
- $FOTSZorbaManifestPath)
- }
- }
- catch err:FODC0002
- {
- error($err:code,
- $err:description,
- concat("
Please make sure the passed 'fotsPath' points to the ",
- "exact location of the FOTS catalog.xml:
",
- resolve-uri($FOTSCatalogFilePath),
- "
and that the passed 'fotsZorbaManifestPath' points to",
- " a file in the same folder as cli.xq:
",
- resolve-uri($FOTSZorbaManifestPath)))
+ {
+ variable $results := driver:run-fots($FOTSCatalogFilePath,
+ $FOTSZorbaManifestPath,
+ (),
+ $exceptedTestSets,
+ (),
+ '',
+ (),
+ fn:false(),
+ $expectedFailuresPath,
+ 'run-test-sets',
+ fn:false());
+
+ file:write("results.xml",
+ $results,
+ $util:writeXML);
+
+ reporting:W3C-reporting($results,
+ $FOTSZorbaManifestPath)
}
};
@@ -175,7 +151,7 @@
variable $W3CTemplate := parse-xml(file:read-text($W3CTemplatePath));
- (: add dependecies:)
+ (: add dependencies:)
(insert nodes
for $dependency in $FOTSZorbaManifest/fots:test-suite-result/fots:dependency
return <dependency type="{$dependency/@type}"
@@ -337,28 +313,3 @@
error($err:code, $err:description)
}
};
-
-declare function reporting:regressions(
-) as xs:string*
-{
- let $old_report:=fn:parse-xml(file:read-text('/home/spungi/work/zorba/repo/fots-ctest/build/bin/report_04_Dec.xml'))
- let $new_report:=fn:parse-xml(file:read-text('/home/spungi/work/zorba/repo/fots-ctest/build/bin/report_18_Dec.xml'))
-
- for $testSetOld in $old_report/*:report/*:test-set
- let $testSetNew := $new_report/*:report/*:test-set[@name = data($testSetOld/@name)]
- let $regression := if (exists($testSetNew))
- then xs:decimal(data($testSetOld/@noFailures)) - xs:decimal(data($testSetNew/@noFailures))
- else xs:decimal(0)
- let $testsOld as xs:string* := tokenize(data($testSetOld/@failedTestNames),",")
- let $testsNew as xs:string* := tokenize(data($testSetNew/@failedTestNames),",")
- where $regression < xs:decimal(0)
- order by $regression ascending
- return
- concat(data($testSetOld/@name),
- " ",
- $regression,
- " ",
- string-join((functx:value-except($testsNew, $testsOld)),","),
- "
")
-};
-
Follow ups
