zorba-coders team mailing list archive

[Nicolae Brinza <nbrinza@xxxxxxxxx>]

Nicolae Brinza has proposed merging lp:~zorba-coders/zorba/process-2 into lp:zorba/process-module.

Commit message:
Version 2.0 of the process module, allows running executables directly, without invoking bash/cmd.exe

Requested reviews:
  Nicolae Brinza (nbrinza)
  Matthias Brantner (matthias-brantner)

For more details, see:
https://code.launchpad.net/~zorba-coders/zorba/process-2/+merge/164354

Version 2.0 of the process module, allows running executables directly, without invoking bash/cmd.exe
-- 
https://code.launchpad.net/~zorba-coders/zorba/process-2/+merge/164354
Your team Zorba Coders is subscribed to branch lp:zorba/process-module.

=== modified file 'src/com/zorba-xquery/www/modules/CMakeLists.txt'
--- src/com/zorba-xquery/www/modules/CMakeLists.txt	2011-07-01 09:24:09 +0000
+++ src/com/zorba-xquery/www/modules/CMakeLists.txt	2013-05-17 12:29:26 +0000
@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-DECLARE_ZORBA_MODULE (URI "http://www.zorba-xquery.com/modules/process"; VERSION 1.0 FILE "process.xq")
+DECLARE_ZORBA_MODULE (URI "http://www.zorba-xquery.com/modules/process"; VERSION 2.0 FILE "process.xq")

=== modified file 'src/com/zorba-xquery/www/modules/process.xq'
--- src/com/zorba-xquery/www/modules/process.xq	2012-12-06 02:13:28 +0000
+++ src/com/zorba-xquery/www/modules/process.xq	2013-05-17 12:29:26 +0000
@@ -28,14 +28,17 @@
  :
  : Potential result:
  : <pre class="ace-static" ace-mode="xquery"><![CDATA[
- : <result xmlns="http://www.zorba-xquery.com/modules/process";>
- :   <stdout>myfile.txt</stout>
- :   <stderr/>
- :   <exit-code>0</exit-code>
- : </result>
+ : {
+ :   "exit-code": 0,
+ :   "stdout": "myfile.txt",
+ :   "stderr": ""
+ : }
  : ]]></pre>
  :
- : @author Cezar Andrei
+ : The exec-command() set of functions allows execution of commands through the operating
+ : system's command line interpreter, such as "sh" on Linux or "cmd.exe" on Windows. 
+ :
+ : @author Cezar Andrei, Nicolae Brinza
  : @project Zorba/IO/Process
  :
  :)
@@ -44,7 +47,100 @@
 declare namespace an = "http://www.zorba-xquery.com/annotations";;
 
 declare namespace ver = "http://www.zorba-xquery.com/options/versioning";;
-declare option ver:module-version "1.0";
+declare option ver:module-version "2.0";
+
+
+(:~
+ : Executes the specified program in a separate process.
+ : This function does not allow arguments to be passed to
+ : the command. The $filename parameter can contain the full path to the 
+ : executable. On Linux systems,  if the specified filename does not contain 
+ : a slash "/", the function duplicates the actions of the shell in searching 
+ : for an executable file. The file is sought in the colon-separated list of 
+ : directory pathnames specified in the PATH environment variable. If this 
+ : variable isn't defined, the path list defaults to the current directory 
+ : followed by the list of directories returned by the operating system.
+ :
+ : @param $filename the name of program to be executed 
+ :
+ : @return the result of the execution as an object as
+ :         shown in the documentation of this module. The exit-code
+ :         returns the exit code of the child process.
+ : For POSIX compliant platforms: returns the program exit code. If the program is 
+ : terminated or stopped: 128 + termination signal code. 
+ : For Windows platforms: returns the return value of the program or the exit 
+ : or terminate process specified value.
+ :
+ : @error process:PROC01 if an error occurred while communicating 
+ :   with the executed process.
+ :)
+declare %an:sequential function process:exec(
+  $filename as xs:string
+) as object() external;
+
+(:~
+ : Executes the specified program in a separate process. 
+ : The $filename parameter can contain the full path to the 
+ : executable. On Linux systems,  if the specified filename does not contain 
+ : a slash "/", the function duplicates the actions of the shell in searching 
+ : for an executable file. The file is sought in the colon-separated list of 
+ : directory pathnames specified in the PATH environment variable. If this 
+ : variable isn't defined, the path list defaults to the current directory 
+ : followed by the list of directories returned by the operating system.
+ : The $args parameters will be passed to the executable file as arguments.
+ :
+ : @param $filename the name of program to be executed 
+ : @param $args arguments to be passed to the executable 
+ :
+ : @return the result of the execution as an object as
+ :         shown in the documentation of this module. The exit-code
+ :         returns the exit code of the child process.
+ : For POSIX compliant platforms: returns the program exit code. If the program is 
+ : terminated or stopped: 128 + termination signal code. 
+ : For Windows platforms: returns the return value of the program or the exit 
+ : or terminate process specified value.
+ :
+ : @error process:PROC01 if an error occurred while communicating 
+ :   with the executed process.
+ :)
+declare %an:sequential function process:exec(
+  $filename as xs:string,
+  $args as xs:string*
+) as object() external;
+
+(:~
+ : Executes the specified program in a separate process. 
+ : The $filename parameter can contain the full path to the 
+ : executable. On Linux systems,  if the specified filename does not contain 
+ : a slash "/", the function duplicates the actions of the shell in searching 
+ : for an executable file. The file is sought in the colon-separated list of 
+ : directory pathnames specified in the PATH environment variable. If this 
+ : variable isn't defined, the path list defaults to the current directory 
+ : followed by the list of directories returned by the operating system.
+ : The $args parameters will be passed to the executable file as arguments.
+ : The $env allows defining and passing environment variables to the target 
+ : process. They should be in the form "ENVVAR=value" where "ENVVAR" is the 
+ : name of the environment variable and "value' is the string value to set it to.
+ :
+ : @param $filename the name of program to be executed 
+ : @param $args arguments to be passed to the executable 
+ :
+ : @return the result of the execution as an object as
+ :         shown in the documentation of this module. The exit-code
+ :         returns the exit code of the child process.
+ : For POSIX compliant platforms: returns the program exit code. If the program is 
+ : terminated or stopped: 128 + termination signal code. 
+ : For Windows platforms: returns the return value of the program or the exit 
+ : or terminate process specified value.
+ :
+ : @error process:PROC01 if an error occurred while communicating 
+ :   with the executed process.
+ :)
+declare %an:sequential function process:exec(
+  $filename as xs:string,
+  $args as xs:string*,
+  $env as xs:string*
+) as object() external;
 
 (:~
  : Executes the specified string command in a separate process.
@@ -64,9 +160,9 @@
  : @error process:PROC01 if an error occurred while communicating 
  :   with the executed process.
  :)
-declare %an:sequential function process:exec(
+declare %an:sequential function process:exec-command(
   $cmd as xs:string
-) as element(process:result) external;
+) as object() external;
 
 (:~
  : Executes the specified string command in a separate process.
@@ -87,7 +183,7 @@
  : @error process:PROC01 if an error occurred while communicating 
  :   with the executed process.
  :)
-declare %an:sequential function process:exec(
+declare %an:sequential function process:exec-command(
   $cmd as xs:string,
   $args as xs:string*
-) as element(process:result) external;
+) as object() external;

=== modified file 'src/com/zorba-xquery/www/modules/process.xq.src/process.cpp'
--- src/com/zorba-xquery/www/modules/process.xq.src/process.cpp	2012-12-05 17:29:38 +0000
+++ src/com/zorba-xquery/www/modules/process.xq.src/process.cpp	2013-05-17 12:29:26 +0000
@@ -100,6 +100,28 @@
   aFactory->createTextNode(lExitCode, lExitCodeString.str());
 }
 
+void create_result_object(
+    zorba::Item&        aResult,
+    const std::string&  aStandardOut,
+    const std::string&  aErrorOut,
+    int                 aExitCode,
+    zorba::ItemFactory* aFactory)
+{  
+  std::vector<std::pair<zorba::Item,zorba::Item> > pairs;
+  
+  pairs.push_back(std::pair<zorba::Item,zorba::Item>(aFactory->createString("exit-code"), aFactory->createInt(aExitCode)));
+  pairs.push_back(std::pair<zorba::Item,zorba::Item>(aFactory->createString("stdout"), aFactory->createString(aStandardOut)));
+  pairs.push_back(std::pair<zorba::Item,zorba::Item>(aFactory->createString("stderr"), aFactory->createString(aErrorOut)));
+  
+  aResult = aFactory->createJSONObject(pairs);  
+}
+
+void free_char_vector(std::vector<char*> argv)
+{
+  for (unsigned int i=0; i<argv.size(); i++)
+    free(argv[i]);    
+}
+
 #ifdef WIN32
 
 /***********************************************
@@ -307,7 +329,7 @@
 #define READ  0
 #define WRITE 1
 
-pid_t zorba_popen(const char *command, int *infp, int *outfp, int *errfp)
+pid_t exec_helper(int *infp, int *outfp, int *errfp, const char *command, char* argv[], char* env[])
 {
     int p_stdin[2];
     int p_stdout[2];
@@ -332,7 +354,13 @@
       close(p_stderr[READ]);
       dup2(p_stderr[WRITE], 2); // duplicate stderr
 
-      execl("/bin/sh", "sh", "-c", command, NULL);
+      if (command)
+        execl("/bin/sh", "sh", "-c", command, NULL);
+      else if (env == NULL)      
+        execvp(argv[0], argv);
+      else
+        execvpe(argv[0], argv, env);      
+        
       perror("execl"); // output the result to standard error
       exit(errno);
     }
@@ -358,10 +386,23 @@
         
     return pid;
 }
+
 #endif
 
+
 /******************************************************************************
  *****************************************************************************/
+String ExecFunction::getOneStringArgument (const Arguments_t& aArgs, int aPos) const
+{
+  Item lItem;
+  Iterator_t  args_iter = aArgs[aPos]->getIterator();
+  args_iter->open();
+  args_iter->next(lItem);
+  zorba::String lTmpString = lItem.getStringValue();
+  args_iter->close();
+  return lTmpString;
+}
+
 zorba::ItemSequence_t
 ExecFunction::evaluate(
   const Arguments_t& aArgs,
@@ -370,6 +411,7 @@
 {
   std::string lCommand;
   std::vector<std::string> lArgs;
+  std::vector<std::string> lEnv;
   int exit_code = 0;
 
   lCommand = getOneStringArgument(aArgs, 0).c_str();
@@ -379,12 +421,20 @@
     zorba::Item lArg;
     Iterator_t arg1_iter = aArgs[1]->getIterator();
     arg1_iter->open();
-    while (arg1_iter->next(lArg))
-    {
+    while (arg1_iter->next(lArg))    
       lArgs.push_back(lArg.getStringValue().c_str());
-    }
     arg1_iter->close();
   }
+  
+  if (aArgs.size() > 2)
+  {
+    zorba::Item lArg;
+    Iterator_t arg1_iter = aArgs[2]->getIterator();
+    arg1_iter->open();
+    while (arg1_iter->next(lArg))    
+      lEnv.push_back(lArg.getStringValue().c_str());
+    arg1_iter->close();    
+  }
 
   std::ostringstream lTmp;
 
@@ -432,18 +482,38 @@
   int errfp;
   int status;
   pid_t pid;
+  
+  std::vector<char*> argv(lArgs.size()+2, NULL);
+  std::vector<char*> env(lEnv.size()+1, NULL);
 
-  pid = zorba_popen(lTmp.str().c_str(), NULL, &outfp, &errfp);
-  if ( pid == -1 )
-  {
-    std::stringstream lErrorMsg;
-    lErrorMsg << "Failed to execute the command (" << pid << ")";
-    Item lQName = ProcessModule::getItemFactory()->createQName(
-      "http://www.zorba-xquery.com/modules/process";, "PROC01");
-    throw USER_EXCEPTION(lQName, lErrorMsg.str().c_str());
-  }
-  else
-  {
+  try
+  {
+    if (theIsExecProgram)
+    {
+      argv[0] = strdup(lCommand.c_str());
+      for (unsigned int i=0; i<lArgs.size(); i++)
+        argv[i+1] = strdup(lArgs[i].c_str()); 
+      
+      for (unsigned int i=0; i<lEnv.size(); i++)
+        env[i] = strdup(lEnv[i].c_str());
+      
+      pid = exec_helper(NULL, &outfp, &errfp, NULL, argv.data(), lEnv.size() ? env.data() : NULL);
+    }
+    else
+    {
+      pid = exec_helper(NULL, &outfp, &errfp, lTmp.str().c_str(), argv.data(), NULL);
+    }
+    
+    if ( pid == -1 )
+    {
+      std::stringstream lErrorMsg;
+      lErrorMsg << "Failed to execute the command (" << pid << ")";
+      Item lQName = ProcessModule::getItemFactory()->createQName(
+            "http://www.zorba-xquery.com/modules/process";, "PROC01");
+      throw USER_EXCEPTION(lQName, lErrorMsg.str().c_str());
+      return NULL;
+    }
+  
     char lBuf[PATH_MAX];
     ssize_t length = 0;
     while ( (length=read(outfp, lBuf, PATH_MAX)) > 0 )
@@ -504,28 +574,23 @@
     }
     
     //std::cout << " exit_code : " << exit_code << std::endl; std::cout.flush();
-
+    free_char_vector(argv);
+    free_char_vector(env);
+  }
+  catch (...)
+  {
+    free_char_vector(argv);
+    free_char_vector(env);
+    throw;
   }
 #endif // WIN32
 
   zorba::Item lResult;
-  create_result_node(lResult, lStdout.str(), lStderr.str(), exit_code,
-                     theModule->getItemFactory());
-
+  create_result_object(lResult, lStdout.str(), lStderr.str(), exit_code,
+                       theModule->getItemFactory());  
   return zorba::ItemSequence_t(new zorba::SingletonItemSequence(lResult));
 }
 
-String ExecFunction::getOneStringArgument (const Arguments_t& aArgs, int aPos)
-  const
-{
-  Item lItem;
-  Iterator_t  args_iter = aArgs[aPos]->getIterator();
-  args_iter->open();
-  args_iter->next(lItem);
-  zorba::String lTmpString = lItem.getStringValue();
-  args_iter->close();
-  return lTmpString;
-}
 
 /******************************************************************************
  *****************************************************************************/
@@ -545,10 +610,14 @@
   zorba::ExternalFunction*& lFunc = theFunctions[aLocalname];
   if (lFind == theFunctions.end())
   {
-    if (!aLocalname.compare("exec"))
+    if (aLocalname.compare("exec-command") == 0)
     {
       lFunc = new ExecFunction(this);
     }
+    else if (aLocalname.compare("exec") == 0)
+    {
+      lFunc = new ExecFunction(this, true);
+    }
   }
   return lFunc;
 }

=== modified file 'src/com/zorba-xquery/www/modules/process.xq.src/process.h'
--- src/com/zorba-xquery/www/modules/process.xq.src/process.h	2012-07-21 01:09:37 +0000
+++ src/com/zorba-xquery/www/modules/process.xq.src/process.h	2013-05-17 12:29:26 +0000
@@ -74,12 +74,13 @@
 class ExecFunction : public ContextualExternalFunction
 {
 public:
-  ExecFunction(const ProcessModule* aModule) : theModule(aModule) {}
+  ExecFunction(const ProcessModule* aModule, bool aExecProgram = false) 
+    : theModule(aModule), theIsExecProgram(aExecProgram) {}
 
   virtual ~ExecFunction() {}
 
   virtual zorba::String
-  getLocalName() const { return "exec"; }
+  getLocalName() const { if (theIsExecProgram) return "exec"; else return "exec-command"; }
 
   virtual zorba::ItemSequence_t
   evaluate(const Arguments_t&,
@@ -93,12 +94,14 @@
 
 protected:
   const ProcessModule* theModule;
+  
+  bool theIsExecProgram; // if set to true, will use the execvpe() version of the system function
+                         // if set to false, will build a command string and pass it to 
+                         // either "bash" or "cmd.exe" (through execl() on Linux)
 
   String getOneStringArgument (const Arguments_t& aArgs, int index) const;
 };
 
-
-
 } /* namespace processmodule */
 } /* namespace zorba */
 

=== added file 'test/ExpQueryResults/process2-01.xml.res'
--- test/ExpQueryResults/process2-01.xml.res	1970-01-01 00:00:00 +0000
+++ test/ExpQueryResults/process2-01.xml.res	2013-05-17 12:29:26 +0000
@@ -0,0 +1,1 @@
+true
\ No newline at end of file

=== added file 'test/ExpQueryResults/process2-02.xml.res'
--- test/ExpQueryResults/process2-02.xml.res	1970-01-01 00:00:00 +0000
+++ test/ExpQueryResults/process2-02.xml.res	2013-05-17 12:29:26 +0000
@@ -0,0 +1,1 @@
+true
\ No newline at end of file

=== added file 'test/ExpQueryResults/process2-03.xml.res'
--- test/ExpQueryResults/process2-03.xml.res	1970-01-01 00:00:00 +0000
+++ test/ExpQueryResults/process2-03.xml.res	2013-05-17 12:29:26 +0000
@@ -0,0 +1,1 @@
+true
\ No newline at end of file

=== added file 'test/ExpQueryResults/process2-04.xml.res'
--- test/ExpQueryResults/process2-04.xml.res	1970-01-01 00:00:00 +0000
+++ test/ExpQueryResults/process2-04.xml.res	2013-05-17 12:29:26 +0000
@@ -0,0 +1,1 @@
+true
\ No newline at end of file

=== added file 'test/ExpQueryResults/process2-05.xml.res'
--- test/ExpQueryResults/process2-05.xml.res	1970-01-01 00:00:00 +0000
+++ test/ExpQueryResults/process2-05.xml.res	2013-05-17 12:29:26 +0000
@@ -0,0 +1,1 @@
+true
\ No newline at end of file

=== modified file 'test/Queries/process.xq'
--- test/Queries/process.xq	2011-08-13 00:08:53 +0000
+++ test/Queries/process.xq	2013-05-17 12:29:26 +0000
@@ -2,22 +2,22 @@
 
 
 {
-  variable $stdOutTest := proc:exec("echo","hello world") ;
-  variable $stdErrTest := proc:exec("echo","Ooops. an error. 1>&amp;2");
-  variable $stdOutWinTest := proc:exec("cmd", ("/c", "echo","hello world")) ;
-  variable $stdErrWinTest := proc:exec("cmd", ("/c", "echo","Ooops. an error. 1>&amp;2"));
+  variable $stdOutTest := proc:exec-command("echo","hello world") ;
+  variable $stdErrTest := proc:exec-command("echo","Ooops. an error. 1>&amp;2");
+  variable $stdOutWinTest := proc:exec-command("cmd", ("/c", "echo","hello world")) ;
+  variable $stdErrWinTest := proc:exec-command("cmd", ("/c", "echo","Ooops. an error. 1>&amp;2"));
   
   let $result :=
     <result>
-      <out>{normalize-space(data($stdOutTest/proc:stdout))}</out>
-      <err>{normalize-space(data($stdErrTest/proc:stderr))}</err>
+      <out>{normalize-space(data($stdOutTest("stdout")))}</out>
+      <err>{normalize-space(data($stdErrTest("stderr")))}</err>
     </result>
   return 
     if (contains($result/err/text(),"is not recognized as an internal or external command"))
     then
       <result>
-        <out>{normalize-space(data($stdOutWinTest/proc:stdout))}</out>
-        <err>{normalize-space(data($stdErrWinTest/proc:stderr))}</err>
+        <out>{normalize-space(data($stdOutWinTest("stdout")))}</out>
+        <err>{normalize-space(data($stdErrWinTest("stderr")))}</err>
       </result>
     else
       $result

=== added file 'test/Queries/process2-01.xq'
--- test/Queries/process2-01.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/process2-01.xq	2013-05-17 12:29:26 +0000
@@ -0,0 +1,5 @@
+import module namespace proc = "http://www.zorba-xquery.com/modules/process";; 
+
+let $result := proc:exec("echo")
+return $result("stdout") eq "
+"

=== added file 'test/Queries/process2-02.xq'
--- test/Queries/process2-02.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/process2-02.xq	2013-05-17 12:29:26 +0000
@@ -0,0 +1,5 @@
+import module namespace proc = "http://www.zorba-xquery.com/modules/process";; 
+
+let $result := proc:exec("echo",("hello","world"))
+return $result("stdout") eq "hello world
+"

=== added file 'test/Queries/process2-03.xq'
--- test/Queries/process2-03.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/process2-03.xq	2013-05-17 12:29:26 +0000
@@ -0,0 +1,5 @@
+import module namespace proc = "http://www.zorba-xquery.com/modules/process";; 
+
+let $result := proc:exec("printenv",("TEST_ENV_VAR"),"TEST_ENV_VAR=foo")
+return $result("stdout") eq "foo
+"

=== added file 'test/Queries/process2-04.xq'
--- test/Queries/process2-04.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/process2-04.xq	2013-05-17 12:29:26 +0000
@@ -0,0 +1,6 @@
+import module namespace proc = "http://www.zorba-xquery.com/modules/process";; 
+
+let $result := proc:exec("printenv",("TEST_ENV_VAR","VAR2"),("TEST_ENV_VAR=foo","VAR2=bar"))
+return $result("stdout") eq "foo
+bar
+"

=== added file 'test/Queries/process2-05.xq'
--- test/Queries/process2-05.xq	1970-01-01 00:00:00 +0000
+++ test/Queries/process2-05.xq	2013-05-17 12:29:26 +0000
@@ -0,0 +1,5 @@
+import module namespace proc = "http://www.zorba-xquery.com/modules/process";; 
+
+let $result := proc:exec("echo","{}[]()()''~!@#$%^&amp;*_-+|<>/?,.")
+return $result("stdout") eq "{}[]()()''~!@#$%^&amp;*_-+|<>/?,.
+"