zorba-coders team mailing list archive
-
zorba-coders team
-
Mailing list archive
-
Message #01365
[Merge] lp:~zorba-coders/zorba/conformance_reports_generation into lp:zorba
Sorin Marian Nasoi has proposed merging lp:~zorba-coders/zorba/conformance_reports_generation into lp:zorba.
Requested reviews:
Chris Hillery (ceejatec)
For more details, see:
https://code.launchpad.net/~zorba-coders/zorba/conformance_reports_generation/+merge/80297
Added testdriver_mt as a option for generating the w3c_testsuite results for W3C.
Also the tests are sorted by name in the XML conformance results in order to make the diffs more readable.
--
https://code.launchpad.net/~zorba-coders/zorba/conformance_reports_generation/+merge/80297
Your team Zorba Coders is subscribed to branch lp:zorba.
=== modified file 'test/rbkt/Scripts/w3c/Readme.txt'
--- test/rbkt/Scripts/w3c/Readme.txt 2011-07-29 21:49:28 +0000
+++ test/rbkt/Scripts/w3c/Readme.txt 2011-10-25 06:33:27 +0000
@@ -14,19 +14,28 @@
#
To generate a report for submitting to the W3C:
-0. For submitting reports to W3C one should compile Zorba with ZORBA_WITH_BIG_INTEGER=ON.
- In order to generate the report for XQueryX compile Zorba by setting:
- ZORBA_XQUERYX=ON and ZORBA_TEST_XQUERYX=ON. These options are set to OFF by default.
+0. For submitting reports to W3C one should compile Zorba with:
+ ZORBA_WITH_BIG_INTEGER=ON
+ ZORBA_XQUERYX=ON
+ ZORBA_TEST_XQUERYX=ON
+ All these options are set to OFF by default.
1. Import XQTS/XQFTTS.
-2. Run a CTest dashboard from your build directory:
+2. Use the "testdriver" to do the testing.
+ Run a CTest dashboard from your build directory:
- ctest -T test -R w3c_testsuite/XQuery
+ ctest -T test -R w3c_testsuite/XQuery or
ctest -T test -R w3c_testsuite/XQueryX
- ctest -T test -R w3c_full_text_testsuite/XQuery
+ ctest -T test -R w3c_full_text_testsuite/XQuery or
ctest -T test -R w3c_full_text_testsuite/XQueryX
+or, *ONLY* for XQTS you can use the "testdriver_mt" to do the testing:
+
+ from the build/test/rbkt execute
+ ./testdriver_mt -b w3c_testsuite/XQuery -w3c or
+ ./testdriver_mt -b w3c_testsuite/XQueryX -w3c
+
3. From this directory (test/rbkt/Scripts/w3c), modify
generate-submission-xqts.xq/generate-submission-xqftts.xq to reflect:
=== modified file 'test/rbkt/Scripts/w3c/Submit_xqts.cmake'
--- test/rbkt/Scripts/w3c/Submit_xqts.cmake 2011-07-29 06:36:14 +0000
+++ test/rbkt/Scripts/w3c/Submit_xqts.cmake 2011-10-25 06:33:27 +0000
@@ -25,12 +25,30 @@
message (FATAL_ERROR "Zorba is required; not found. Specify -DZORBA_BUILD_DIR to point to your build directory if necessary.")
endif ()
-# Read most recent CTest tag
-file (STRINGS "${builddir}/Testing/TAG" _tag_content)
-list (GET _tag_content 0 ctest_tag)
-set (testfile "${builddir}/Testing/${ctest_tag}/Test.xml")
-
-# Execute Zorba
-execute_process (COMMAND "${zorba}" --omit-xml-declaration --indent
- --query "${cwd}/generate-submission-xqts.xq" --as-files
- -e "ctests=${testfile}")
+find_program(testdriver_mt NAMES testdriver_mt PATHS "${builddir}" "${builddir}/release" PATH_SUFFIXES test/rbkt)
+if (testdriver_mt)
+ # Use the Test.xml generated by the testdriver_mt
+ set (testfile "${builddir}/Testing/Test.xml")
+
+ if(testfile)
+ # Execute Zorba
+ execute_process (COMMAND "${zorba}" --omit-xml-declaration --indent
+ --query "${cwd}/generate-submission-xqts.xq" --as-files
+ -e "ctests=${testfile}")
+ else()
+ message (FATAL_ERROR "Testdriver_mt was found, but ${builddir}/Testing/Test.xml was not found: please run 'testdriver_mt -b w3c_testsuite -w3c' in order to generate the 'Test.xml' report first.")
+ endif ()
+
+else ()
+ # Read most recent CTest tag and find out the corresponding Test.xml
+ file (STRINGS "${builddir}/Testing/TAG" _tag_content)
+ list (GET _tag_content 0 ctest_tag)
+ set (testfile "${builddir}/Testing/${ctest_tag}/Test.xml")
+ # MESSAGE(STATUS "using testresults from: ${testfile}")
+
+ # Execute Zorba
+ execute_process (COMMAND "${zorba}" --omit-xml-declaration --indent
+ --query "${cwd}/generate-submission-xqts.xq" --as-files
+ -e "ctests=${testfile}")
+
+endif ()
=== modified file 'test/rbkt/Scripts/w3c/generate-submission-xqftts.xq'
--- test/rbkt/Scripts/w3c/generate-submission-xqftts.xq 2011-10-07 08:28:43 +0000
+++ test/rbkt/Scripts/w3c/generate-submission-xqftts.xq 2011-10-25 06:33:27 +0000
@@ -72,6 +72,7 @@
{
for $test in $ctests/*:Site/*:Testing/*:Test
let $testname := fn:tokenize(fn:data($test/*:Name), "/")[last()]
+ order by $testname
return
<test-case
name="{$testname}"
=== modified file 'test/rbkt/Scripts/w3c/generate-submission-xqts.xq'
--- test/rbkt/Scripts/w3c/generate-submission-xqts.xq 2011-10-07 08:28:43 +0000
+++ test/rbkt/Scripts/w3c/generate-submission-xqts.xq 2011-10-25 06:33:27 +0000
@@ -64,6 +64,7 @@
{
for $test in $ctests/*:Site/*:Testing/*:Test
let $testname := fn:tokenize(fn:data($test/*:Name), "/")[last()]
+ order by $testname
return
if (fn:contains(fn:data($test),'StaticTyping')) then
<test-case
=== modified file 'test/rbkt/testdriver_mt.cpp'
--- test/rbkt/testdriver_mt.cpp 2011-07-16 23:46:39 +0000
+++ test/rbkt/testdriver_mt.cpp 2011-10-25 06:33:27 +0000
@@ -1,12 +1,12 @@
/*
* Copyright 2006-2008 The FLWOR Foundation.
- *
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -229,7 +229,7 @@
/*******************************************************************************
Create all the directories in a filepath, if thehy don't exist already, and
- make sure the file can be created/opened.
+ make sure the file can be created/opened.
********************************************************************************/
void createPath(const fs::path& filePath, std::ofstream& fileStream)
{
@@ -238,19 +238,19 @@
{
fs::path dirPath = filePath;
dirPath = dirPath.remove_leaf();
-
+
if (!fs::exists(dirPath.file_string()))
{
fs::create_directories(dirPath.file_string());
// clear the bad flag on windows, which for some unknown reason doesn't reset when opening a file again
- fileStream.clear();
+ fileStream.clear();
fileStream.open(filePath.file_string().c_str());
}
if (!fileStream.good())
{
- std::cerr << "Could not open file: "
+ std::cerr << "Could not open file: "
<< filePath.file_string() << std::endl;
abort();
}
@@ -261,9 +261,9 @@
/*******************************************************************************
********************************************************************************/
-bool checkErrors(const Specification& lSpec, const TestDiagnosticHandler& errHandler, std::ostream& lOutput)
+bool checkErrors(const Specification& lSpec, const TestDiagnosticHandler& errHandler, std::ostream& lOutput)
{
- if (isErrorExpected(errHandler, &lSpec))
+ if (isErrorExpected(errHandler, &lSpec))
{
printErrors(errHandler,
"The following execution errors occurred as expected",
@@ -350,7 +350,7 @@
zorba::XQuery_t query;
- // Choose a query to run. If no query is available, the thread finishes.
+ // Choose a query to run. If no query is available, the thread finishes.
// To choose the next query, the whole query container must be locked.
// After the query is chosen, we release the global container lock and
// acquire the query-specific lock for the chosen query. The query lock
@@ -393,7 +393,7 @@
// exprected errors, or the pathnames of reference-result files.
specPath = fs::change_extension(queryPath, ".spec");
if (fs::exists(specPath))
- querySpec.parseFile(specPath.file_string(), rbkt_src_dir, rbkt_bin_dir);
+ querySpec.parseFile(specPath.file_string(), rbkt_src_dir, rbkt_bin_dir);
// Get the pathnames of the ref-result files found in the .spec file (if any).
// If no ref-results file was specified in the .spec file, create a default
@@ -402,14 +402,14 @@
// from the path
for (std::vector<std::string>::const_iterator iter = querySpec.resultsBegin();
iter != querySpec.resultsEnd();
- ++iter)
+ ++iter)
{
fs::path refFilePath(*iter);
refFileSpecified = true;
refFilePaths.push_back(refFilePath);
}
- if (refFilePaths.size() == 0)
+ if (refFilePaths.size() == 0)
{
std::string relativeRefFile = relativeQueryFile;
if (queries->theIsW3Cbucket)
@@ -424,7 +424,7 @@
fs::path refFilePath = fs::path(queries->theRefsDir) / (relativeRefFile);
refFilePath = fs::change_extension(refFilePath, ".xml.res");
- if (fs::exists(refFilePath))
+ if (fs::exists(refFilePath))
refFileSpecified = true;
refFilePaths.push_back(refFilePath);
@@ -451,7 +451,7 @@
// resolvers in the static context.
zorba::StaticContext_t sctx = zorba->createStaticContext();
- if (queries->theIsW3Cbucket)
+ if (queries->theIsW3Cbucket)
{
mmapper.reset
(new zorba::TestModuleURIMapper(mod_map_file.c_str(),
@@ -497,7 +497,7 @@
errHandler.setErrorFile(errorFilePath.file_string());
//
- // Compile the query, if it has not been compiled already.
+ // Compile the query, if it has not been compiled already.
//
if (queries->theQueryObjects[queryNo] == 0)
{
@@ -555,7 +555,7 @@
//
// Clone the compiled query and register with it the error handler of the
// current thread.
- //
+ //
query = queries->theQueryObjects[queryNo]->clone();
query->registerDiagnosticHandler(&errHandler);
@@ -607,7 +607,7 @@
goto done;
}
}
- else if (querySpec.getComparisonMethod() != "Ignore"
+ else if (querySpec.getComparisonMethod() != "Ignore"
&& querySpec.errorsSize() > 0 && !refFileSpecified)
{
queries->theOutput << "FAILURE : thread " << tno << " query " << queryNo
@@ -626,16 +626,16 @@
failure = true;
goto done;
}
- else
+ else
{
bool foundRefFile = false;
ulong i;
- for (i = 0; i < refFilePaths.size(); i++)
+ for (i = 0; i < refFilePaths.size(); i++)
{
std::string refFilePath = refFilePaths[i].file_string();
std::string resFilePath = resultFilePath.file_string();
- int lLine, lCol;
+ int lLine, lCol;
std::string lRefLine, lResultLine;
bool success = zorba::fileEquals(refFilePath.c_str(), resFilePath.c_str(),
lLine, lCol,
@@ -704,9 +704,10 @@
std::cerr << "\nusage: testdriver_mt -b <bucket> [options] OR" << std::endl
<< " testdriver_mt -p <path-to-bucket> [options]" << std::endl
<< "Options:" << std::endl
- << " [-t <numThreads>] [-n <runsPerQuery>] [-e test-name] [-q]" << std::endl
+ << " [-t <numThreads>] [-n <runsPerQuery>] [-e test-name] [-q] [-w3c]" << std::endl
<< " [-k <known-failures file>] [-o <report logfile>]" << std::endl
- << " -q - Quiet; only a summary report will be displayed." << std::endl;
+ << " -q - Quiet; only a summary report will be displayed." << std::endl
+ << " -w3c means that the output also contains a XML report that can be used in the W3C conformance reports generation." << std::endl;
exit(1);
}
@@ -727,10 +728,13 @@
std::string refsDir;
std::string modulePath;
std::string reportFilename = "mt.log";
+ std::string XMLreportFilename = "Test.xml";
std::string reportFilepath;
+ std::string XMLreportFilepath;
std::string knownFailuresFilepath;
bool haveKnownFailures = false;
bool quiet = false;
+ bool generateW3CData = false;
fs::path path;
std::set<std::string> knownFailures;
@@ -744,7 +748,7 @@
// Parse the arg list
long arg = 1;
-
+
while (arg < argc)
{
if (!strcmp(argv[arg], "-t"))
@@ -764,7 +768,7 @@
else if (!strcmp(argv[arg], "-b"))
{
arg++;
- bucketName = argv[arg];
+ bucketName = argv[arg];
}
else if (!strcmp(argv[arg], "-p"))
{
@@ -779,7 +783,7 @@
else if (!strcmp(argv[arg], "-o"))
{
arg++;
- reportFilename = argv[arg];
+ reportFilename = argv[arg];
}
else if (!strcmp(argv[arg], "-k"))
{
@@ -791,6 +795,10 @@
{
quiet = true;
}
+ else if (!strcmp(argv[arg], "-w3c"))
+ {
+ generateW3CData = true;
+ }
else if (!strcmp(argv[arg], "--module-path"))
{
arg++;
@@ -804,7 +812,7 @@
arg++;
}
- if (bucketName == "" && bucketPath == "")
+ if (bucketName == "" && bucketPath == "")
{
usage();
}
@@ -858,6 +866,10 @@
}
reportFilepath = zorba::RBKT_BINARY_DIR + "/../../Testing/" + reportFilename;
+ if(generateW3CData)
+ {
+ XMLreportFilepath = zorba::RBKT_BINARY_DIR + "/../../Testing/" + XMLreportFilename;
+ }
//
// Make sure the directories exist. For the results dir, if it doesn't exist,
@@ -924,7 +936,7 @@
queries.theQueryFilenames.push_back(relativeQueryFile);
}
}
- else
+ else
{
queries.theQueryFilenames = testNames;
}
@@ -976,7 +988,7 @@
// shutdown
delete[] threads;
-#else
+#else
// Windows code
HANDLE* threads = new HANDLE[numThreads]; // pthread_t* threads = new pthread_t[numThreads];
@@ -989,7 +1001,7 @@
}
WaitForMultipleObjects(numThreads, threads, TRUE, INFINITE);
-
+
// shutdown
delete[] threads;
delete[] thread_ids;
@@ -1002,18 +1014,18 @@
//
// If known-failures file specified, load it into a map for quick lookups.
//
- if (haveKnownFailures)
+ if (haveKnownFailures)
{
queries.theOutput << std::endl << "Loading known failures file "
<< knownFailuresFilepath << "..." << std::endl;
std::ifstream knownFailuresFile(knownFailuresFilepath.c_str());
- while (knownFailuresFile.good())
+ while (knownFailuresFile.good())
{
std::string testname;
getline(knownFailuresFile, testname);
knownFailures.insert(testname);
}
- queries.theOutput << "Loaded " << knownFailures.size()
+ queries.theOutput << "Loaded " << knownFailures.size()
<< " known failures" << std::endl << std::endl;
}
@@ -1023,7 +1035,7 @@
// on regressions/progressions or not. Also keep a running list of
// failed tests.
//
- std::ostringstream report;
+ std::ostringstream report, XMLreport;
std::ostringstream failedTests;
long numFailures = 0;
long numRegressions = 0;
@@ -1036,7 +1048,7 @@
fs::path queryPath = fs::path(pathPrefix) / (queries.theQueryFilenames[i]);
fs::path queryName = fs::change_extension(queryPath, "");
bool queryWasKnownToFail = false;
- if (haveKnownFailures)
+ if (haveKnownFailures)
{
queryWasKnownToFail = (knownFailures.count(queryName.file_string()) != 0);
}
@@ -1056,6 +1068,7 @@
{
report << i << ":" << queryName.file_string() << std::endl;
}
+
}
else
{
@@ -1066,27 +1079,38 @@
<< queryName.file_string() << std::endl;
}
}
+ if(generateW3CData)
+ {
+ std::string status = (queries.theQueryStates[i] == true)?"pass":"fail";
+ XMLreport << "<Test Status='" << status << "'><Name>" << queryName.file_string() << "</Name></Test>" << std::endl;
+ }
}
std::ofstream reportFile(reportFilepath.c_str());
reportFile << report.str();
+ if(generateW3CData)
+ {
+ std::ofstream w3cReportFile(XMLreportFilepath.c_str());
+ w3cReportFile << "<?xml version='1.0' encoding='UTF-8'?>" << std::endl << "<Site><Testing>" << std::endl << XMLreport.str() << "</Testing></Site>" << std::endl;
+ }
+
// Don't use theOutput here - this is the summary we always want to
// see
- std::cout << "Number of queries run = "
- << queries.theNumQueries << std::endl << "Number of failures = "
+ std::cout << "Number of queries run = "
+ << queries.theNumQueries << std::endl << "Number of failures = "
<< numFailures << std::endl;
if (haveKnownFailures)
{
std::cout << "Number of regressions = " << numRegressions << std::endl;
std::cout << "Number of progressions = " << numProgressions << std::endl;
- if (report.str().length() > 0)
+ if (report.str().length() > 0)
{
std::cout << std::endl << "Individual test reports:" << std::endl
<< report.str() << std::endl;
}
- if (numRegressions == 0 && numProgressions > 0)
+ if (numRegressions == 0 && numProgressions > 0)
{
std::cout << "No regressions and some progressions!" << std::endl
<< "Updating known failures file "
@@ -1101,7 +1125,7 @@
std::cout << std::endl << "Failed queries: " << std::endl
<< report.str() << std::endl;
}
-
+
queries.clear();
zorba->shutdown();
zorba::StoreManager::shutdownStore(store);
=== modified file 'test/update/Scripts/generate-submission-xquts.xq'
--- test/update/Scripts/generate-submission-xquts.xq 2011-10-07 08:28:43 +0000
+++ test/update/Scripts/generate-submission-xquts.xq 2011-10-25 06:33:27 +0000
@@ -64,6 +64,7 @@
{
for $test in $ctests/*:Site/*:Testing/*:Test
let $testname := fn:tokenize(fn:data($test/*:Name), "/")[last()]
+ order by $testname
return
if(fn:exists(fn:index-of(('revalidation-declaration-01-fail','revalidation-declaration-03-fail','revalidation-declaration-05-fail','revalidate-valtrans-ins-003'
,'fn-put-003-fail','fn-put-004-fail','fn-put-005-fail','fn-put-006-fail'),$testname))) then
Follow ups
