zorba-coders team mailing list archive

[Chris Hillery <chillery+launchpad@xxxxxxxxx>]

Chris Hillery has proposed merging lp:~zorba-coders/zorba/fots-ctest into lp:zorba.

Commit message:
CTest integration for FOTS driver. Allows automated import of FOTS, and enabling one CTest test per FOTS test set *or* per FOTS test case.

Requested reviews:
  Sorin Marian Nasoi (sorin.marian.nasoi)

For more details, see:
https://code.launchpad.net/~zorba-coders/zorba/fots-ctest/+merge/138071
-- 
https://code.launchpad.net/~zorba-coders/zorba/fots-ctest/+merge/138071
Your team Zorba Coders is subscribed to branch lp:zorba.

=== modified file 'CMakeLists.txt'
--- CMakeLists.txt	2012-10-08 12:09:36 +0000
+++ CMakeLists.txt	2012-12-05 09:16:21 +0000
@@ -550,12 +550,16 @@
 ADD_SUBDIRECTORY(scripts)
 ADD_SUBDIRECTORY(cmake_modules)
 
-# Subdirectory ordering: We need to include "test" before "config" so
-# ZorbaConfig knows about testdriver. We need to include "config"
-# before "modules" so external modules will be able to find
-# ZorbaConfig.cmake. We need to include "modules" before "include" so
-# config.h can know eg. whether we found CURL. We need to include "modules"
-# before "doc" so xqdoc can know what non-core modules are available.
+# Subdirectory ordering: We need to include "bin" before "test" so
+# that test/fots can add zorbacmd-based tests. We need to include
+# "test" before "config" so ZorbaConfig knows about testdriver. We
+# need to include "config" before "modules" so external modules will
+# be able to find ZorbaConfig.cmake. We need to include "modules"
+# before "include" so config.h can know eg. whether we found CURL. We
+# need to include "modules" before "doc" so xqdoc can know what
+# non-core modules are available.
+
+ADD_SUBDIRECTORY(bin)
 
 ADD_SUBDIRECTORY(test)
 ADD_SUBDIRECTORY(config)
@@ -572,8 +576,6 @@
   ADD_SUBDIRECTORY (swig)
 ENDIF (SWIG_FOUND)
 
-ADD_SUBDIRECTORY(bin)
-
 # add the uninstall support
 # QQQ move to config/
 CONFIGURE_FILE(

=== modified file 'bin/CMakeLists.txt'
--- bin/CMakeLists.txt	2012-11-08 21:57:10 +0000
+++ bin/CMakeLists.txt	2012-12-05 09:16:21 +0000
@@ -15,6 +15,8 @@
 INCLUDE_DIRECTORIES(AFTER ${CMAKE_SOURCE_DIR}/src/)
 INCLUDE_DIRECTORIES(AFTER ${CMAKE_CURRENT_SOURCE_DIR})
 
+INCLUDE("${CMAKE_SOURCE_DIR}/cmake_modules/ZorbaGenerateExe.cmake")
+
 IF (ZORBA_WITH_DEBUGGER)
   IF (NOT WIN32)
     FIND_PACKAGE (Libedit)
@@ -49,7 +51,6 @@
   path_util.cpp
 )
 
-INCLUDE("${CMAKE_SOURCE_DIR}/cmake_modules/ZorbaGenerateExe.cmake")
 ZORBA_GENERATE_EXE("zorbacmd" "${SRCS}" "" "zorba" "bin")
 
 

=== modified file 'test/CMakeLists.txt'
--- test/CMakeLists.txt	2012-10-08 12:09:36 +0000
+++ test/CMakeLists.txt	2012-12-05 09:16:21 +0000
@@ -37,6 +37,8 @@
 ADD_SUBDIRECTORY(update)
 ADD_SUBDIRECTORY(sax2)
 
+ADD_SUBDIRECTORY(fots)
+
 SET (TEST_PLAN_SERIALIZER_SRCS
     test_plan_serializer.cpp
     timer.cpp)

=== added directory 'test/fots'
=== added file 'test/fots/CMakeLists.txt'
--- test/fots/CMakeLists.txt	1970-01-01 00:00:00 +0000
+++ test/fots/CMakeLists.txt	2012-12-05 09:16:21 +0000
@@ -0,0 +1,57 @@
+# Copyright 2006-2008 The FLWOR Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Add a target to import FOTS
+ADD_CUSTOM_TARGET(fots-import
+  COMMAND "${CMAKE_COMMAND}" 
+  -D "ZORBA=${ZORBA_EXE}"
+  -D "BUILDDIR=${PROJECT_BINARY_DIR}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/ImportFOTS.cmake")
+
+# Create cached version of activate setting
+SET (ZORBA_FOTS_ACTIVATE inactive CACHE STRING "Which FOTS tests to activate")
+MARK_AS_ADVANCED (ZORBA_FOTS_ACTIVATE)
+
+# Add targets to set activate flag
+ADD_CUSTOM_TARGET(fots-activate-sets
+  COMMAND "${CMAKE_COMMAND}"
+  -D "ZORBA_FOTS_ACTIVATE=sets" "${PROJECT_SOURCE_DIR}"
+  WORKING_DIRECTORY "${PROJECT_BINARY_DIR}")
+ADD_CUSTOM_TARGET(fots-activate-cases
+  COMMAND "${CMAKE_COMMAND}"
+  -D "ZORBA_FOTS_ACTIVATE=cases" "${PROJECT_SOURCE_DIR}"
+  WORKING_DIRECTORY "${PROJECT_BINARY_DIR}")
+ADD_CUSTOM_TARGET(fots-deactivate
+  COMMAND "${CMAKE_COMMAND}"
+  -D "ZORBA_FOTS_ACTIVATE=inactive" "${PROJECT_SOURCE_DIR}"
+  WORKING_DIRECTORY "${PROJECT_BINARY_DIR}")
+
+# Load FOTS test scripts (if they exist) depending on flag
+IF ("${ZORBA_FOTS_ACTIVATE}" STREQUAL "sets")
+  MESSAGE (STATUS "Adding FOTS test-set tests")
+  SET (_fotsscript "${CMAKE_CURRENT_BINARY_DIR}/FOTSSetTests.cmake")
+ELSEIF ("${ZORBA_FOTS_ACTIVATE}" STREQUAL "cases")
+  MESSAGE (STATUS "Adding FOTS test-case tests (may take a while...)")
+  SET (_fotsscript "${CMAKE_CURRENT_BINARY_DIR}/FOTSCaseTests.cmake")
+ELSE ("${ZORBA_FOTS_ACTIVATE}" STREQUAL "sets")
+  SET (_fotsscript "")
+ENDIF ("${ZORBA_FOTS_ACTIVATE}" STREQUAL "sets")
+IF (NOT _fotsscript STREQUAL "")
+  IF (EXISTS "${_fotsscript}")
+    INCLUDE ("${_fotsscript}")
+  ELSE (EXISTS "${_fotsscript}")
+    MESSAGE (WARNING "...${_fotsscript} does not exist, not adding FOTS tests "
+      "after all (did you forget to run the 'fots-import' target?)")
+  ENDIF (EXISTS "${_fotsscript}")
+ENDIF (NOT _fotsscript STREQUAL "")

=== added file 'test/fots/ImportFOTS.cmake'
--- test/fots/ImportFOTS.cmake	1970-01-01 00:00:00 +0000
+++ test/fots/ImportFOTS.cmake	2012-12-05 09:16:21 +0000
@@ -0,0 +1,99 @@
+# Copyright 2006-2008 The FLWOR Foundation.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# Downloads the latest FOTS archive and prepares for CTest integration.
+# Required argument: BUILDDIR (should be ${PROJECT_BINARY_DIR})
+# Required argument: ZORBA (full path to Zorba command-line executable)
+
+
+# Change this to publish updated FOTS archives
+SET (FOTS_ARCHIVE "FOTS_041212.tgz")
+
+
+# Download and unpack pre-created archive
+SET (_outdir "${BUILDDIR}/test/fots")
+MESSAGE (STATUS "Downloading FOTS archive...")
+FILE (DOWNLOAD http://zorbatest.lambda.nu:8080/~ceej/${FOTS_ARCHIVE}
+  "${_outdir}/${FOTS_ARCHIVE}" STATUS _dlstat SHOW_PROGRESS)
+LIST (GET _dlstat 0 _dlcode)
+IF (NOT _dlcode EQUAL 0)
+  LIST (GET _dlstat 1 _dlmsg)
+  MESSAGE (FATAL_ERROR "Error downloading FOTS archive: ${_dlmsg}")
+ENDIF (NOT _dlcode EQUAL 0)
+
+MESSAGE (STATUS "Unpacking FOTS archive...")
+EXECUTE_PROCESS (COMMAND "${CMAKE_COMMAND}" -E tar xf ${FOTS_ARCHIVE}
+  WORKING_DIRECTORY "${_outdir}" RESULT_VARIABLE _unpackstat)
+IF (NOT _unpackstat EQUAL 0)
+  MESSAGE (FATAL_ERROR "Error unpackaging FOTS archive: ${_unpackstat}")
+ENDIF (NOT _unpackstat EQUAL 0)
+
+# Scan FOTS catalogs and create ADD_TEST scripts. First, create a list
+# of test sets. We assume that the FOTS driver is in a sibling directory
+# named fots_driver.
+MESSAGE (STATUS "Creating CTest scripts...")
+EXECUTE_PROCESS (COMMAND "${ZORBA}" --omit-xml-declaration -f -q
+  "${CMAKE_CURRENT_LIST_DIR}/../fots_driver/cli.xq"
+  -e "fotsPath:=${_outdir}/2011/QT3-test-suite/catalog.xml"
+  -e mode:=list-test-sets
+  OUTPUT_VARIABLE _testsetsstring RESULT_VARIABLE _zstat)
+IF (NOT _zstat EQUAL 0)
+  MESSAGE (FATAL_ERROR "Zorba failed with message: ${_zstat}\n"
+    "(perhaps you haven't built Zorba yet?)")
+ENDIF (NOT _zstat EQUAL 0)
+STRING (REGEX REPLACE "[\n\r]+" ";" _testsets ${_testsetsstring})
+
+SET (_setscript "${_outdir}/FOTSSetTests.cmake")
+SET (_casescript "${_outdir}/FOTSCaseTests.cmake")
+FILE (REMOVE "${_setscript}")
+FILE (REMOVE "${_casescript}")
+
+# Iterate through all test sets.
+FOREACH (_testset ${_testsets})
+  # Add a test for the whole test set.
+  FILE (APPEND "${_setscript}"
+    "ZORBA_ADD_TEST (FOTS-set-${_testset} zorbacmd -f -q "
+    "${CMAKE_CURRENT_LIST_DIR}/../fots_driver/cli.xq "
+    "-e fotsPath:=${_outdir}/2011/QT3-test-suite/catalog.xml "
+    "-e mode:=run-test-sets -e testSetPrefixes:=${_testset} "
+    "--disable-http-resolution --indent)\n"
+    "ZORBA_SET_TEST_PROPERTY (FOTS-set-${_testset} "
+    "FAIL_REGULAR_EXPRESSION \"result=\\\"fail\\\"\")\n")
+
+  # Iterate through the test cases for this test set.
+#  EXECUTE_PROCESS (COMMAND "${ZORBA}" --omit-xml-declaration -f -q
+#    "${CMAKE_CURRENT_LIST_DIR}/../fots_driver/cli.xq"
+#    -e "fotsPath:=${_outdir}/2011/QT3-test-suite/catalog.xml"
+#    -e mode:=list-test-cases -e testSetPrefixes:=${_testset}
+#    OUTPUT_VARIABLE _testcasesstring)
+#  STRING (REGEX REPLACE "[\n\r]+" ";" _testcases ${_testcasesstring})
+  EXECUTE_PROCESS (COMMAND "${ZORBA}" --omit-xml-declaration -f -q
+    "${CMAKE_CURRENT_LIST_DIR}/get-tests.xq"
+    --base-uri "${_outdir}/2011/QT3-test-suite/catalog.xml"
+    -e testSet:=${_testset}
+    OUTPUT_VARIABLE _testcases)
+  FOREACH (_testcase ${_testcases})
+    # Add a test for the test case.
+    FILE (APPEND "${_casescript}"
+      "ZORBA_ADD_TEST (FOTS-${_testset}-${_testcase} zorbacmd -f -q "
+      "${CMAKE_CURRENT_LIST_DIR}/../fots_driver/cli.xq "
+      "-e fotsPath:=${_outdir}/2011/QT3-test-suite/catalog.xml "
+      "-e mode:=run-test-case "
+      "-e testSetName:=${_testset} -e testCaseName:=${_testcase} "
+      "--disable-http-resolution --indent)\n"
+      "ZORBA_SET_TEST_PROPERTY (FOTS-${_testset}-${_testcase} "
+      "PASS_REGULAR_EXPRESSION \"test-case name=\\\"${_testcase}\\\" result=\\\"pass\\\"\")\n")
+  ENDFOREACH (_testcase)
+ENDFOREACH (_testset)

=== added file 'test/fots/README.TXT'
--- test/fots/README.TXT	1970-01-01 00:00:00 +0000
+++ test/fots/README.TXT	2012-12-05 09:16:21 +0000
@@ -0,0 +1,26 @@
+This directory contains scripts for integrating the Zorba FOTS Driver
+(contained in the directory ../fots_driver) with CTest.
+
+It adds four targets to the Zorba project: one to download and import
+FOTS, and three to configure which kinds of CTests are added to the
+project.
+
+  fots-import: This target will download FOTS and create CTest
+  scripts. This is somewhat slow (around a minute), but you only need
+  to do it once. You should not need to repeat this target unless you
+  delete your build directory, or you want to update to a newer FOTS.
+
+  fots-activate-sets: This target will re-configure your project to
+  include one CTest test for each FOTS test set (roughly 370 tests).
+  This is the recommended approach for running FOTS tests as it
+  provides a good tradeoff between testing time and test granularity.
+
+  fots-activate-cases: This target will re-configure your project to
+  include one CTest test for each FOTS test case (roughly 27000
+  tests). This is only recommended when debugging a problem reported
+  with a specific test case, because CTest really isn't designed for
+  such a large number of tests; it will slow down considerably.
+
+  fots-deactivate: This target will re-configure your project to
+  include no CTest tests for FOTS. Use this when doing normal
+  development.

=== added file 'test/fots/get-tests.xq'
--- test/fots/get-tests.xq	1970-01-01 00:00:00 +0000
+++ test/fots/get-tests.xq	2012-12-05 09:16:21 +0000
@@ -0,0 +1,25 @@
+(:
+ : Copyright 2006-2011 The FLWOR Foundation.
+ :
+ : Licensed under the Apache License, Version 2.0 (the "License");
+ : you may not use this file except in compliance with the License.
+ : You may obtain a copy of the License at
+ :
+ : http://www.apache.org/licenses/LICENSE-2.0
+ :
+ : Unless required by applicable law or agreed to in writing, software
+ : distributed under the License is distributed on an "AS IS" BASIS,
+ : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ : See the License for the specific language governing permissions and
+ : limitations under the License.
+ :)
+
+declare namespace c = "http://www.w3.org/2010/09/qt-fots-catalog";;
+
+declare variable $fotsPath as xs:string external := "";
+declare variable $testSet as xs:string external := "";
+
+let $catalog := fn:doc(fn:static-base-uri())/c:catalog
+let $test-set := $catalog/c:test-set[@name = $testSet]
+let $test-relfile := fn:trace(data($test-set/@file), "Loading")
+return fn:string-join(data(fn:doc($test-relfile)/c:test-set/c:test-case/@name), ';')