zorba-coders team mailing list archive
-
zorba-coders team
-
Mailing list archive
-
Message #26516
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
Paul J. Lucas has proposed merging lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba.
Commit message:
1. New FTP client.
2. Moved curl_streambuf code to shared util-curl module.
Requested reviews:
Paul J. Lucas (paul-lucas)
For more details, see:
https://code.launchpad.net/~zorba-coders/zorba/feature-ftp-client/+merge/201082
1. New FTP client.
2. Moved curl_streambuf code to shared util-curl module.
--
https://code.launchpad.net/~zorba-coders/zorba/feature-ftp-client/+merge/201082
Your team Zorba Coders is subscribed to branch lp:zorba.
=== modified file 'CMakeCPack.cmake'
--- CMakeCPack.cmake 2013-08-06 16:51:59 +0000
+++ CMakeCPack.cmake 2014-01-09 19:12:12 +0000
@@ -128,7 +128,7 @@
SET(CPACK_POSTFLIGHT_SCRIPT "${CMAKE_BINARY_DIR}/osx_postflight.sh")
CONFIGURE_FILE("${CMAKE_SOURCE_DIR}/scripts/osx_postflight.sh.in"
"${CMAKE_BINARY_DIR}/osx_postflight.sh")
- MESSAGE ( STATUS "script = "${CPACK_POSTFLIGHT_SCRIPT} )
+ MESSAGE ( STATUS "script = ${CPACK_POSTFLIGHT_SCRIPT}" )
ENDIF ( APPLE )
INCLUDE(CPack)
INCLUDE(CPack.cmake)
=== modified file 'CMakeLists.txt'
--- CMakeLists.txt 2013-11-04 22:29:39 +0000
+++ CMakeLists.txt 2014-01-09 19:12:12 +0000
@@ -235,6 +235,24 @@
ENDIF(_clock_gettime_in_rt)
#
+# cURL
+#
+#/*IF (ZORBA_SUPPRESS_CURL)*/
+#/* MESSAGE( STATUS "ZORBA_SUPPRESS_CURL is true - not searching for cURL library" )*/
+#/*ELSE (ZORBA_SUPPRESS_CURL)*/
+#/* MESSAGE( STATUS "Looking for cURL" )*/
+#/* FIND_PACKAGE( CURL )*/
+#/* IF (CURL_FOUND)*/
+#/* MESSAGE( STATUS "Found cURL library -- " ${CURL_LIBRARIES})*/
+#/* INCLUDE_DIRECTORIES( ${CURL_INCLUDE_DIR} )*/
+#/* SET(requiredlibs ${requiredlibs} ${CURL_LIBRARIES})*/
+#/* ELSE (CURL_FOUND)*/
+#/* MESSAGE( STATUS "The cURL library was not found - ftp-client and http-client will not be built" )*/
+#/* ENDIF (CURL_FOUND)*/
+#/*ENDIF (ZORBA_SUPPRESS_CURL)*/
+#/*SET( ZORBA_HAVE_CURL ${CURL_FOUND} CACHE BOOL "Whether Zorba found cURL" FORCE )*/
+
+#
# TcMalloc support
#
IF(ZORBA_USE_TCMALLOC)
@@ -591,7 +609,6 @@
# non-core modules are available.
ADD_SUBDIRECTORY(bin)
-
ADD_SUBDIRECTORY(test)
ADD_SUBDIRECTORY(config)
ADD_SUBDIRECTORY(schemas)
=== modified file 'include/zorba/item.h'
--- include/zorba/item.h 2013-09-26 07:38:44 +0000
+++ include/zorba/item.h 2014-01-09 19:12:12 +0000
@@ -418,7 +418,7 @@
* @throw ZorbaException if an error occured (e.g. the Item is not of type JSON Object).
*/
Item
- getObjectValue(String aName) const;
+ getObjectValue(String const &aName) const;
/**
* Checks whether the item's content is streamable.
=== modified file 'include/zorba/util/base64_stream.h'
--- include/zorba/util/base64_stream.h 2013-08-05 22:23:16 +0000
+++ include/zorba/util/base64_stream.h 2014-01-09 19:12:12 +0000
@@ -269,11 +269,10 @@
* @param stream The stream to attach the base64::streambuf to. If the
* stream already has a base64::streambuf attached to it, this contructor
* does nothing.
- * @param charset The name of the character encoding to convert from/to.
* @return \c true only if a base64::streambuf was attached.
*/
- bool attach( StreamType &stream, char const *charset ) {
- if ( base64::attach( stream, charset ) ) {
+ bool attach( StreamType &stream ) {
+ if ( base64::attach( stream ) ) {
stream_ = &stream;
return true;
}
=== renamed file 'src/util/mem_streambuf.h' => 'include/zorba/util/mem_streambuf.h'
--- src/util/mem_streambuf.h 2013-08-01 00:31:20 +0000
+++ include/zorba/util/mem_streambuf.h 2014-01-09 19:12:12 +0000
@@ -14,8 +14,8 @@
* limitations under the License.
*/
-#ifndef ZORBA_MMAP_STREAMBUF_H
-#define ZORBA_MMAP_STREAMBUF_H
+#ifndef ZORBA_API_MEM_STREAMBUF_H
+#define ZORBA_API_MEM_STREAMBUF_H
#include <streambuf>
@@ -118,5 +118,5 @@
///////////////////////////////////////////////////////////////////////////////
} // namespace zorba
-#endif /* ZORBA_MMAP_STREAMBUF_H */
+#endif /* ZORBA_API_MEM_STREAMBUF_H */
/* vim:set et sw=2 ts=2: */
=== modified file 'modules/CMakeLists.txt'
--- modules/CMakeLists.txt 2013-10-16 22:04:45 +0000
+++ modules/CMakeLists.txt 2014-01-09 19:12:12 +0000
@@ -12,11 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+ADD_SUBDIRECTORY(util-curl)
ADD_SUBDIRECTORY(atomic)
ADD_SUBDIRECTORY(com)
ADD_SUBDIRECTORY(full-text)
ADD_SUBDIRECTORY(functx)
ADD_SUBDIRECTORY(dateTime)
+ADD_SUBDIRECTORY(ftp-client)
ADD_SUBDIRECTORY(http-client)
ADD_SUBDIRECTORY(item)
ADD_SUBDIRECTORY(json)
=== added directory 'modules/ftp-client'
=== added file 'modules/ftp-client/CMakeLists.txt'
--- modules/ftp-client/CMakeLists.txt 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/CMakeLists.txt 2014-01-09 19:12:12 +0000
@@ -0,0 +1,26 @@
+# Copyright 2006-2013 The FLWOR Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+IF (ZORBA_HAVE_CURL)
+ SET(ZorbaUtilCurlModule_DIR "../util-curl")
+ FIND_PACKAGE(ZorbaUtilCurlModule REQUIRED)
+ IF (ZorbaUtilCurlModule_FOUND)
+ INCLUDE_DIRECTORIES("${ZorbaUtilCurlModule_INCLUDE_DIRS}")
+ DECLARE_ZORBA_MODULE( FILE ftp-client.xq VERSION 1.0
+ URI "http://zorba.io/modules/ftp-client";
+ LINK_LIBRARIES ${CURL_LIBRARIES} ${ZorbaUtilCurlModule_LIBS})
+ ENDIF (ZorbaUtilCurlModule_FOUND)
+ENDIF (ZORBA_HAVE_CURL)
+
+# vim:set et sw=2 ts=2:
=== added file 'modules/ftp-client/ftp-client.xq'
--- modules/ftp-client/ftp-client.xq 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/ftp-client.xq 2014-01-09 19:12:12 +0000
@@ -0,0 +1,342 @@
+jsoniq version "1.0";
+
+(:
+ : Copyright 2006-2013 The FLWOR Foundation.
+ :
+ : Licensed under the Apache License, Version 2.0 (the "License");
+ : you may not use this file except in compliance with the License.
+ : You may obtain a copy of the License at
+ :
+ : http://www.apache.org/licenses/LICENSE-2.0
+ :
+ : Unless required by applicable law or agreed to in writing, software
+ : distributed under the License is distributed on an "AS IS" BASIS,
+ : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ : See the License for the specific language governing permissions and
+ : limitations under the License.
+ :)
+
+(:===========================================================================:)
+
+(:~
+ : This module provides functions for performing FTP commands.
+ : <p/>
+ : @author Paul J. Lucas
+ : @library <a href="http://curl.haxx.se/";>cURL Library</a>
+ : @project Zorba/Input Output/FTP Client
+ :)
+module namespace ftp = "http://zorba.io/modules/ftp-client";;
+
+declare namespace an = "http://zorba.io/annotations";;
+declare namespace ver = "http://zorba.io/options/versioning";;
+
+declare option ver:module-version "1.0";
+
+(:~
+ : Attempts to connect to an FTP server specified by the given URI.
+ :
+ : @param $uri The address of the FTP server to connect to.
+ : It may either be simple host-name
+ : (<code>ftp.example.com</code>)
+ : or a URI using the <code>ftp</code>
+ : or <code>ftps</code> (FTP over SSL/TLS)
+ : schemes
+ : (<code>ftp://ftp.example.com</code>).
+ : If the latter,
+ : the URI may also contain
+ : <code>username</code>,
+ : <code>password</code>,
+ : and
+ : <code>port</code>
+ : authority subcomponents
+ : per RFC 3986.
+ : @param $options The options to use:
+ : <dl>
+ : <dt><code>user</code></dt>
+ : <dd>
+ : The user to log in as;
+ : default: <code>"ftp"</code> (for anonymous FTP).
+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
+ : the user, if specified, must be part of the URI
+ : and the value of this option is ignored.
+ : </dd>
+ : <dt><code>password</code></dt>
+ : <dd>
+ : The password to use to log in;
+ : default: <code>"ftp@xxxxxxxxxxx"</code> (for anonymous FTP).
+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
+ : the password, if specified, must be part of the URI
+ : and the value of this option is ignored.
+ : </dd>
+ : <dt><code>port</code></dt>
+ : <dd>
+ : The port number to use;
+ : default: whatever the default for the protocol is.
+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
+ : the port, if specified, must be part of the URI
+ : and the value of this option is ignored.
+ : </dd>
+ : <dt><code>protocol</code></dt>
+ : <dd>
+ : The protocol to use,
+ : either <code>"ftp'</code> or <code>"ftps"</code>;
+ : default: <code>"ftp"</code>.
+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
+ : the protocol, if specified, must be part of the URI
+ : and the value of this option is ignored.
+ : </dd>
+ : <dt><code>SSL-communication</code></dt>
+ : <dd>
+ : Whether to use SSL/TLS, one of:
+ : <dl>
+ : <dt><code>"none"</code></dt>
+ : <dd>Don't use SSL.</dd>
+ : <dt><code>"try"</code></dt>
+ : <dd>Try using SSL, but, if unsuccessful, continue anyway.</dd>
+ : <dt><code>"control"</code></dt>
+ : <dd>Require SSL for the control connection.</dd>
+ : <dt><code>"all"</code></dt>
+ : <dd>Require SSL for all communication.</dd>
+ : </dl>
+ : default:
+ : <code>"none"</code> for protocol <code>"ftp"</code>
+ : and
+ : <code>"all"</code> for protocol <code>"ftps"</code>
+ : Note that any value other than <code>"none"</code>
+ : for <code>"ftp"</code> implies <em>explicit</em> SSL;
+ : use of <code>"ftps"</code> implies <em>implicit</em> SSL.
+ : See "<a href="http://en.wikipedia.org/wiki/FTPS#Methods_of_invoking_security";>Methods of invoking security</a>."
+ : </dd>
+ : <dt><code>SSL-verify</code></dt>
+ : <dd>
+ : When doing FTP over SSL/TLS,
+ : whether to verify the authenticity of the server's certificate
+ : and that the certificate is for that server;
+ : default: <code>true</code>.
+ : (You should <em>never</em> set this to <code>false</code>
+ : unless you are testing your own FTP server
+ : with a self-signed certificate.)
+ : </dd>
+ : <dt><code>trace</code></dt>
+ : <dd>
+ : Whether to emit information to standard error
+ : tracing the communication between the FTP client and server;
+ : default: <code>false</code>.
+ : </dd>
+ : </dl>
+ : @return an opaque URI that serves as a connection handle to be used with
+ : other functions in this module.
+ : @error ftp:ALREADY_CONNECTED if <code>$uri</code> is already connected to.
+ : @error ftp:INVALID_ARGUMENT if any option is invalid.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:connect( $uri as string, $options as object )
+ as anyURI external;
+
+(:~
+ : Deletes a file from the FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $remote-path The path of the file to delete.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:delete( $conn as string, $remote-path as string )
+ external;
+
+(:~
+ : Disconnects from an FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : After successful completion of of this function, the handle is no longer
+ : valid.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:disconnect( $conn as anyURI )
+ external;
+
+(:~
+ : Gets a binary file from the FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $remote-path The path of the file to get.
+ : @return the binary content of <code>$remote-path</code>.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:get-binary( $conn as anyURI, $remote-path as string )
+ as base64Binary external;
+
+(:~
+ : Gets a text file from the FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $remote-path The path of the file to get. It must not be empty.
+ : @param $encoding The character encoding of the file.
+ : @return the text content of <code>$remote-path</code>.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty
+ : or <code>$encoding</code> is either an invalid or unsupported encoding.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:get-text( $conn as anyURI, $remote-path as string, $encoding as string )
+ as string external;
+
+(:~
+ : Gets a text file, presumed to be encoded in UTF-8, from the FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $remote-path The path of the file to get. It must not be empty.
+ : @return the text content of <code>$remote-path</code>.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:get-text( $conn as anyURI, $remote-path as string )
+ as string
+{
+ ftp:get-text( $conn, $remote-path, "UTF-8" )
+};
+
+(:~
+ : Gets a listing for a directory on an FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $remote-path The full path of the directory on the FTP server to get
+ : the directory listing for.
+ : An empty path is equivalent to <code>/</code>.
+ : @return Returns a sequence of JSON objects, one per file or subdirectory in
+ : the listing.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:list( $conn as anyURI, $remote-path as string )
+ as object* external;
+
+(:~
+ : Creates a directory on the FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $remote-path The path of the new directory to create.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:mkdir( $conn as string, $remote-path as string )
+ external;
+
+(:~
+ : Uploads binary data to a file to the FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $binary The binary data to upload.
+ : @param $remote-path The path of the file to upload to. It must not be empty.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:put-binary( $conn as anyURI, $binary as base64Binary,
+ $remote-path as string )
+ external;
+
+(:~
+ : Uploads text to a file to the FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $text The text to upload.
+ : @param $remote-path The path of the file to upload to. It must not be empty.
+ : @param $encoding The character encoding of the file.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty
+ : or <code>$encoding</code> is either an invalid or unsupported encoding.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:put-text( $conn as anyURI, $text as string, $remote-path as string,
+ $encoding as string )
+ external;
+
+(:~
+ : Uploads text to a UTF-8 encoded file on the FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $text The text to upload.
+ : @param $remote-path The path of the file to upload to. It must not be empty.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:put-text( $conn as anyURI, $text as string, $remote-path as string )
+{
+ ftp:put-text( $conn, $text, $remote-path, "UTF-8" )
+};
+
+(:~
+ : Renames a file or directory on an FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $remote-from-path The path of the file or directory to rename.
+ : @param $remote-to-path The new name.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-from-path</code> or
+ : <code>$remote-to-path</code> is empty.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:rename( $conn as string, $remote-from-path as string,
+ $remote-to-path as string )
+ external;
+
+(:~
+ : Removes a directory from the FTP server.
+ :
+ : @param $conn The opaque URI connection handle previously returned by
+ : <code>ftp:connect()</code>.
+ : @param $remote-path The path of the directory to remove.
+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
+ : or is no longer a valid handle.
+ : @error ftp:FTP_ERROR if there was some other FTP error.
+ :)
+declare %an:sequential function
+ftp:rmdir( $conn as string, $remote-path as string )
+ external;
+
+(:===========================================================================:)
+
+(: vim:set syntax=xquery et sw=2 ts=2: :)
=== added directory 'modules/ftp-client/ftp-client.xq.src'
=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_connections.cpp'
--- modules/ftp-client/ftp-client.xq.src/ftp_connections.cpp 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/ftp-client.xq.src/ftp_connections.cpp 2014-01-09 19:12:12 +0000
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2006-2013 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// standard
+#include <cassert>
+
+// local
+#include "ftp_connections.h"
+
+namespace zorba {
+namespace ftp_client {
+
+///////////////////////////////////////////////////////////////////////////////
+
+connections::~connections() {
+ for ( key_buf_map::const_iterator i = key_buf_.begin();
+ i != key_buf_.end(); ++i ) {
+ delete i->second;
+ }
+}
+
+bool connections::delete_buf( String const &key ) {
+ key_buf_map::iterator const i( key_buf_.find( key ) );
+ if ( i != key_buf_.end() ) {
+ delete i->second;
+ key_buf_.erase( i );
+ return true;
+ }
+ return false;
+}
+
+void connections::destroy() throw() {
+ delete this;
+}
+
+curl::streambuf* connections::get_buf( String const &key ) const {
+ key_buf_map::const_iterator const i( key_buf_.find( key ) );
+ if ( i != key_buf_.end() )
+ return i->second;
+ return 0;
+}
+
+curl::streambuf* connections::new_buf( String const &key ) {
+ curl::streambuf *&buf = key_buf_[ key ];
+ assert( !buf );
+ buf = new curl::streambuf();
+ return buf;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace ftp_client
+} // namespace zorba
+/* vim:set et sw=2 ts=2: */
=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_connections.h'
--- modules/ftp-client/ftp-client.xq.src/ftp_connections.h 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/ftp-client.xq.src/ftp_connections.h 2014-01-09 19:12:12 +0000
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2006-2013 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#ifndef ZORBA_FTP_CLIENT_FTP_CONNECTIONS_H
+#define ZORBA_FTP_CLIENT_FTP_CONNECTIONS_H
+
+// standard
+#include <map>
+
+// Zorba
+#include <zorba/external_function_parameter.h>
+#include <zorba/zorba_string.h>
+
+// util-curl module
+#include "curl_streambuf.h"
+
+namespace zorba {
+namespace ftp_client {
+
+///////////////////////////////////////////////////////////////////////////////
+
+/**
+ * An ftp::connections is-an ExternalFunctionParameter (that's added to a
+ * query's dynamic context) for keeping a mapping between keys (representing
+ * URIs of FTP servers) and their associated curl::streambufs (the active
+ * connections to said servers).
+ */
+class connections : public ExternalFunctionParameter {
+public:
+ ~connections();
+
+ /**
+ * Deletes a curl::streambuf associatied with the given \a key.
+ *
+ * @param key The key associated with the curl::streambuf to delete.
+ * @return Returns \a true only if \a key was associated with a
+ * curl::streambuf and therefore deleted; \c false otherwise.
+ */
+ bool delete_buf( String const &key );
+
+ /**
+ * Gets an existing curl::streambuf associatied with the given \a key.
+ *
+ * @param key The key to get the associated curl::streambuf for.
+ * @return Returns said curl::streambuf or \c nullptr if \a key has no
+ * associated curl::streambuf.
+ */
+ curl::streambuf* get_buf( String const &key ) const;
+
+ /**
+ * Creates a new curl::streambuf and associates it with the given \a key.
+ *
+ * @param key The key to associate the newly created curl::streambuf to.
+ * The key must not alraedy be associated with any curl::streambuf.
+ * @return Returns the newly created curl::streambuf.
+ */
+ curl::streambuf* new_buf( String const &key );
+
+ // inherited
+ virtual void destroy() throw();
+
+private:
+ // map keys -> cURL::streambuf*
+ typedef std::map<String,curl::streambuf*> key_buf_map;
+ key_buf_map key_buf_;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace ftp_client
+} // namespace zorba
+#endif /* ZORBA_FTP_CLIENT_FTP_CONNECTIONS_H */
+/* vim:set et sw=2 ts=2: */
=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_functions.cpp'
--- modules/ftp-client/ftp-client.xq.src/ftp_functions.cpp 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/ftp-client.xq.src/ftp_functions.cpp 2014-01-09 19:12:12 +0000
@@ -0,0 +1,932 @@
+/*
+ * Copyright 2006-2013 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// standard
+#include <cctype>
+#include <ctime>
+#include <istream>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <utility> /* for pair */
+#include <vector>
+
+// Zorba
+#include <zorba/dynamic_context.h>
+#include <zorba/empty_sequence.h>
+#include <zorba/item_factory.h>
+#include <zorba/item_sequence.h>
+#include <zorba/iterator.h>
+#include <zorba/singleton_item_sequence.h>
+#include <zorba/store_consts.h>
+#include <zorba/user_exception.h>
+#include <zorba/util/base64_stream.h>
+#include <zorba/util/mem_streambuf.h>
+#include <zorba/util/transcode_stream.h>
+
+// local
+#include "ftp_connections.h"
+#include "ftp_functions.h"
+#include "ftp_module.h"
+#include "ftpparse.c"
+
+// FTP reply codes; see RFC 959.
+#define FTP_REPLY_ACTION_COMPLETED 250
+#define FTP_REPLY_PATH_CREATED 257
+#define FTP_REPLY_ACTION_ABORTED 451
+#define FTP_REPLY_ACTION_NOT_TAKEN 550
+
+#define IS_ATOMIC_TYPE(ITEM,TYPE) \
+ ( (ITEM).isAtomic() && (ITEM).getTypeCode() == store::TYPE )
+
+// Eliminates "control may reach end of non-void function" warning.
+#define THROW_EXCEPTION(...) while (1) throw_exception( __VA_ARGS__ )
+
+/**
+ * This is our "external function parameter" name in the dynamic context.
+ */
+#define ZORBA_FTP_CONNECTIONS "http://zorba.io/modules/ftp-client/connections";
+
+using namespace std;
+
+namespace zorba {
+namespace ftp_client {
+
+///////////////////////////////////////////////////////////////////////////////
+
+struct curl_helper {
+ curl_helper( curl::streambuf*, curl_slist* = 0 );
+ ~curl_helper();
+private:
+ curl::streambuf *const cbuf_;
+ curl_slist *const slist_;
+};
+
+curl_helper::curl_helper( curl::streambuf *cbuf, curl_slist *slist ) :
+ cbuf_( cbuf ),
+ slist_( slist )
+{
+ ZORBA_CURLM_ASSERT( curl_multi_remove_handle( cbuf_->curlm(), cbuf_->curl() ) );
+}
+
+curl_helper::~curl_helper() {
+ if ( slist_ )
+ curl_slist_free_all( slist_ );
+ CURL *const cobj = cbuf_->curl();
+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, NULL );
+ curl_easy_setopt( cobj, CURLOPT_HEADERDATA, NULL );
+ curl_easy_setopt( cobj, CURLOPT_HEADERFUNCTION, NULL );
+ curl_easy_setopt( cobj, CURLOPT_UPLOAD, 0L );
+ curl_multi_add_handle( cbuf_->curlm(), cobj );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+static connections& get_connections( DynamicContext const *dctx ) {
+ connections *conns = static_cast<connections*>(
+ dctx->getExternalFunctionParameter( ZORBA_FTP_CONNECTIONS )
+ );
+ if ( !conns ) {
+ conns = new connections();
+ dctx->addExternalFunctionParameter( ZORBA_FTP_CONNECTIONS, conns );
+ }
+ return *conns;
+}
+
+static int get_ftp_reply_code( CURL *cobj ) {
+ long code;
+ ZORBA_CURL_ASSERT( curl_easy_getinfo( cobj, CURLINFO_RESPONSE_CODE, &code ) );
+ return static_cast<int>( code );
+}
+
+inline int get_scheme_len( String const &uri ) {
+ if ( uri.compare( 0, 6, "ftp://"; ) == 0 )
+ return 6;
+ if ( uri.compare( 0, 6, "ftps://" ) == 0 )
+ return 7;
+ return 0;
+}
+
+static String host_part( String /* intentionally not const& */ uri ) {
+ if ( int const scheme_len = get_scheme_len( uri ) ) {
+ uri.erase( 0, scheme_len );
+ String::size_type const at_pos = uri.find( '@' );
+ if ( at_pos != String::npos )
+ uri.erase( 0, at_pos + 1 ); // erase user:password@
+ String::size_type const colon_pos = uri.find( ':' );
+ if ( colon_pos != String::npos )
+ uri.erase( colon_pos ); // erase :port
+ }
+ return uri;
+}
+
+static String make_uri( String const &uri,
+ String path, // intentionally not const&
+ bool path_is_dir = false ) {
+ if ( path.empty() )
+ path = '/';
+ else {
+ if ( path_is_dir && path[ path.size() - 1 ] != '/' )
+ path += '/';
+ if ( path[0] != '/' )
+ path.insert( (String::size_type)0, 1, '/' );
+ }
+ String result( uri );
+ result += path;
+ return result;
+}
+
+static void stream_releaser( istream *is ) {
+ delete is;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+function::function( module const *m, char const *local_name ) :
+ module_( m ),
+ local_name_( local_name )
+{
+}
+
+bool function::get_bool_opt( Item const &options, char const *key,
+ bool default_value ) const {
+ Item const item( options.getObjectValue( key ) );
+ if ( item.isNull() )
+ return default_value;
+ if ( !IS_ATOMIC_TYPE( item, XS_BOOLEAN ) )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", key, "value must be boolean" );
+ return item.getBooleanValue();
+}
+
+int function::get_integer_opt( Item const &options, char const *key,
+ int default_value ) const {
+ Item const item( options.getObjectValue( key ) );
+ if ( item.isNull() )
+ return default_value;
+ if ( !IS_ATOMIC_TYPE( item, XS_INTEGER ) )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", key, "value must be integer" );
+ return item.getIntValue();
+}
+
+Item function::get_item_arg( ExternalFunction::Arguments_t const &args,
+ unsigned pos ) const {
+ Item result;
+ if ( pos < args.size() ) {
+ Iterator_t it( args[ pos ]->getIterator() );
+ it->open();
+ it->next( result );
+ it->close();
+ }
+ return result;
+}
+
+String function::getLocalName() const {
+ return local_name_;
+}
+
+String function::get_string_arg( ExternalFunction::Arguments_t const &args,
+ unsigned pos ) const {
+ String s;
+ Item const item( get_item_arg( args, pos ) );
+ if ( !item.isNull() )
+ s = item.getStringValue();
+ return s;
+}
+
+String function::get_string_opt( Item const &options, char const *key,
+ char const *default_value ) const {
+ Item const item( options.getObjectValue( key ) );
+ if ( item.isNull() )
+ return default_value;
+ if ( !IS_ATOMIC_TYPE( item, XS_STRING ) )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", key, "value must be string" );
+ return item.getStringValue();
+}
+
+String function::getURI() const {
+ return module_->getURI();
+}
+
+curl::streambuf* function::require_connection( DynamicContext const *dctx,
+ String const &conn ) const {
+ connections &conns = get_connections( dctx );
+ if ( curl::streambuf *const cbuf = conns.get_buf( conn.c_str() ) )
+ return cbuf;
+ THROW_EXCEPTION( "NOT_CONNECTED", host_part( conn ), "not connnected" );
+}
+
+void function::throw_exception( char const *error_code, char const *object,
+ char const *message, int ftp_code ) const {
+ string s;
+
+ if ( object && *object ) {
+ ostringstream oss;
+ oss << '"' << object << "\": " << message;
+ s = oss.str();
+ } else
+ s = message;
+
+ if ( ftp_code ) {
+ ostringstream oss;
+ oss << " (FTP code " << ftp_code << ')';
+ s += oss.str();
+ }
+
+ throw USER_EXCEPTION(
+ module_->getItemFactory()->createQName( getURI(), error_code ), s
+ );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+get_function::get_function( module const *m, char const *local_name,
+ bool text ) :
+ function( m, local_name ),
+ text_( text )
+{
+}
+
+ItemSequence_t
+get_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const conn( get_string_arg( args, 0 ) );
+ String const path( get_string_arg( args, 1 ) );
+ if ( path.empty() )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
+ String const encoding( text_ ? get_string_arg( args, 2 ) : "" );
+ if ( !encoding.empty() && transcode::is_necessary( encoding.c_str() ) &&
+ !transcode::is_supported( encoding.c_str() ) ) {
+ THROW_EXCEPTION( "INVALID_ARGUMENT", encoding, "unsupported encoding" );
+ }
+ String const uri( make_uri( conn, path ) );
+
+ curl::streambuf *const cbuf = require_connection( dctx, conn );
+ CURL *const cobj = cbuf->curl();
+ curl_easy_setopt( cobj, CURLOPT_TRANSFERTEXT, text_ ? 1 : 0 );
+ curl_easy_setopt( cobj, CURLOPT_URL, uri.c_str() );
+
+ istream *const is = new istream( cbuf );
+ if ( transcode::is_necessary( encoding.c_str() ) )
+ transcode::attach( *is, encoding.c_str() );
+
+ ItemFactory *const f = module_->getItemFactory();
+ Item result(
+ text_ ?
+ f->createStreamableString( *is, &stream_releaser )
+ :
+ f->createStreamableBase64Binary( *is, &stream_releaser, false )
+ );
+ return ItemSequence_t( new SingletonItemSequence( result ) );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+static size_t curl_read_callback( void *ptr, size_t size, size_t nmemb,
+ void *data ) {
+ size *= nmemb;
+ istream *const is = static_cast<istream*>( data );
+ is->read( static_cast<char*>( ptr ), static_cast<streamsize>( size ) );
+ return is->gcount();
+}
+
+put_function::put_function( module const *m, char const *local_name,
+ bool text ) :
+ function( m, local_name ),
+ text_( text )
+{
+}
+
+ItemSequence_t
+put_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const conn( get_string_arg( args, 0 ) );
+ Item put_item( get_item_arg( args, 1 ) );
+ String const path( get_string_arg( args, 2 ) );
+ if ( path.empty() )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
+ String const encoding( text_ ? get_string_arg( args, 2 ) : "" );
+ if ( !encoding.empty() && transcode::is_necessary( encoding.c_str() ) &&
+ !transcode::is_supported( encoding.c_str() ) ) {
+ THROW_EXCEPTION( "INVALID_ARGUMENT", encoding, "unsupported encoding" );
+ }
+ String const uri( make_uri( conn, path ) );
+
+ curl::streambuf *const cbuf = require_connection( dctx, conn );
+ CURL *const cobj = cbuf->curl();
+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_TRANSFERTEXT, text_ ? 1 : 0 ) );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_URL, uri.c_str() ) );
+
+ istream *is;
+ mem_streambuf mbuf;
+ auto_ptr<istream> raii_is;
+ String text;
+
+ if ( put_item.isStreamable() ) {
+ is = &put_item.getStream();
+ } else {
+ if ( text_ ) {
+ text = put_item.getStringValue();
+ mbuf.set( const_cast<char*>( text.data() ), text.size() );
+ } else {
+ size_t size;
+ char const *const data = put_item.getBase64BinaryValue( size );
+ mbuf.set( const_cast<char*>( data ), size );
+ }
+ is = new istream( &mbuf );
+ raii_is.reset( is );
+ }
+
+ //
+ // Thest raii_* objects must be constructed after raii_is so their destructor
+ // is called before that of raii_is (reverse order of construction) so either
+ // the base64_streambuf or transcode_streambuf is destroyed before the stream
+ // it's attached to is.
+ //
+ base64::auto_attach<istream> raii_b64;
+ transcode::auto_attach<istream> raii_xcode;
+ if ( text_ ) {
+ if ( !encoding.empty() && transcode::is_necessary( encoding.c_str() ) )
+ raii_xcode.attach( *is, encoding.c_str() );
+ } else {
+ if ( put_item.isEncoded() )
+ raii_b64.attach( *is );
+ }
+
+ try {
+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_READDATA, is ) );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_READFUNCTION, curl_read_callback ) );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_UPLOAD, 1L ) );
+ curl_helper helper( cbuf );
+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
+ return ItemSequence_t( new EmptySequence() );
+ }
+ catch ( std::exception const &e ) {
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+connect_function::connect_function( module const *m ) :
+ function( m, "connect" )
+{
+}
+
+ItemSequence_t
+connect_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const uri( get_string_arg( args, 0 ) );
+ String conn( uri );
+ //
+ // Even though we allow ftp URIs ("ftp://host";), we limit them to refer only
+ // to the host and not either a file or subdirectory; hence chop off
+ // everything starting at the first '/'.
+ //
+ int const scheme_len = get_scheme_len( conn );
+ String::size_type const slash_pos = conn.find( scheme_len, '/' );
+ if ( slash_pos != String::npos )
+ conn.erase( slash_pos );
+
+ Item const options( get_item_arg( args, 1 ) );
+ String const password( get_string_opt( options, "password" ) );
+ int const port( get_integer_opt( options, "port" ) );
+ String const protocol( get_string_opt( options, "protocol", "ftp" ) );
+ String const ssl_comm( get_string_opt( options, "SSL-communication" ) );
+ bool const ssl_verify( get_bool_opt( options, "SSL-verify", true ) );
+ bool const trace( get_bool_opt( options, "trace", false ) );
+ String const user( get_string_opt( options, "user" ) );
+
+ if ( !(protocol == "ftp" || protocol == "ftps") )
+ THROW_EXCEPTION(
+ "INVALID_ARGUMENT", "protcol", "must be either ftp or ftps"
+ );
+
+ long use_ssl;
+ if ( !ssl_comm.empty() ) {
+ if ( ssl_comm == "all" )
+ use_ssl = CURLUSESSL_ALL;
+ else if ( ssl_comm == "control" )
+ use_ssl = CURLUSESSL_CONTROL;
+ else if ( ssl_comm == "none" )
+ use_ssl = CURLUSESSL_NONE;
+ else if ( ssl_comm == "try" )
+ use_ssl = CURLUSESSL_TRY;
+ else
+ THROW_EXCEPTION(
+ "INVALID_ARGUMENT", "SSL-communication",
+ "must be one of: none, try, control, or all"
+ );
+ } else if ( protocol == "ftps" )
+ use_ssl = CURLUSESSL_ALL;
+ else
+ use_ssl = CURLUSESSL_NONE;
+
+ if ( !scheme_len ) {
+ if ( user.empty() && !password.empty() )
+ THROW_EXCEPTION(
+ "INVALID_ARGUMENT", "", "empty user and non-empty password"
+ );
+ if ( !user.empty() ) {
+ conn.insert( (String::size_type)0, 1, '@' );
+ if ( !password.empty() ) {
+ char *const esc_password =
+ curl_escape( const_cast<char*>( password.data() ), password.size() );
+ conn.insert( 0, esc_password );
+ curl_free( esc_password );
+ conn.insert( (String::size_type)0, 1, ':' );
+ }
+ conn.insert( 0, user );
+ }
+ conn.insert( 0, "://" );
+ conn.insert( 0, protocol );
+ if ( port ) {
+ conn.append( 1, ':' );
+ ostringstream oss;
+ oss << port;
+ conn.append( oss.str() );
+ }
+ }
+
+ connections &conns = get_connections( dctx );
+ curl::streambuf *cbuf = conns.get_buf( conn );
+ if ( cbuf )
+ THROW_EXCEPTION(
+ "ALREADY_CONNECTED", uri, "connection previously established"
+ );
+ cbuf = conns.new_buf( conn );
+
+ try {
+ cbuf->open( conn.c_str() );
+ CURL *const cobj = cbuf->curl();
+
+ if ( trace )
+ cbuf->curl_verbose( true );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_USE_SSL, use_ssl ) );
+ if ( !ssl_verify ) {
+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_SSL_VERIFYHOST, 0L ) );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_SSL_VERIFYPEER, 0L ) );
+ }
+
+ curl_helper helper( cbuf );
+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
+ Item result( module_->getItemFactory()->createAnyURI( conn ) );
+ return ItemSequence_t( new SingletonItemSequence( result ) );
+ }
+ catch ( std::exception const &e ) {
+ THROW_EXCEPTION( "FTP_ERROR", uri, e.what() );
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+#if ZORBA_FTP_CWD_IMPLEMENTED
+ItemSequence_t
+cwd_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const conn( get_string_arg( args, 0 ) );
+ String const path( get_string_arg( args, 1 ) );
+ if ( path.empty() )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
+
+ curl::streambuf *const cbuf = require_connection( dctx, conn );
+ CURL *const cobj = cbuf->curl();
+
+ String req( "CWD " );
+ req += path;
+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, req.c_str() );
+ curl_easy_perform( cobj );
+
+ return ItemSequence_t( new EmptySequence() );
+}
+#endif
+
+///////////////////////////////////////////////////////////////////////////////
+
+delete_function::delete_function( module const *m ) :
+ function( m, "delete" )
+{
+}
+
+ItemSequence_t
+delete_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const conn( get_string_arg( args, 0 ) );
+ String const path( get_string_arg( args, 1 ) );
+ if ( path.empty() )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
+ String const command( "DELE " + path );
+
+ curl::streambuf *const cbuf = require_connection( dctx, conn );
+ CURL *const cobj = cbuf->curl();
+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, command.c_str() );
+
+ try {
+ curl_helper helper( cbuf );
+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
+ }
+ catch ( curl::exception const &e ) {
+ int const ftp_code = get_ftp_reply_code( cobj );
+ switch ( ftp_code ) {
+ case FTP_REPLY_ACTION_COMPLETED:
+ if ( e.curl_code() == CURLE_FTP_COULDNT_RETR_FILE ) {
+ //
+ // After the file is deleted, CURL tries to RETR it which fails, so
+ // ignore that error.
+ // (Also see <http://stackoverflow.com/a/13515807/99089>.)
+ //
+ break;
+ }
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
+ case FTP_REPLY_ACTION_NOT_TAKEN:
+ THROW_EXCEPTION( "FTP_ERROR", path, "file not found", ftp_code );
+ default:
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what(), ftp_code );
+ } // switch
+ }
+ catch ( std::exception const &e ) {
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
+ }
+ return ItemSequence_t( new EmptySequence() );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+disconnect_function::disconnect_function( module const *m ) :
+ function( m, "disconnect" )
+{
+}
+
+ItemSequence_t
+disconnect_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const conn( get_string_arg( args, 0 ) );
+ connections &conns = get_connections( dctx );
+ if ( !conns.delete_buf( conn ) )
+ THROW_EXCEPTION( "NOT_CONNECTED", host_part( conn ), "not connected" );
+ return ItemSequence_t( new EmptySequence() );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+get_binary_function::get_binary_function( module const *m ) :
+ get_function( m, "get-binary", false )
+{
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+get_text_function::get_text_function( module const *m ) :
+ get_function( m, "get-text", true )
+{
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+class list_iterator : public ItemSequence, public Iterator {
+public:
+ list_iterator( curl::streambuf*, ItemFactory* );
+
+ // inherited from ItemSequence
+ Iterator_t getIterator();
+
+ // inherited from Iterator
+ void close();
+ bool isOpen() const;
+ bool next( Item& );
+ void open();
+
+private:
+ ItemFactory *const factory_;
+ istream is_;
+ bool open_;
+};
+
+list_iterator::list_iterator( curl::streambuf *cbuf, ItemFactory *factory ) :
+ factory_( factory ),
+ is_( cbuf ),
+ open_( false )
+{
+}
+
+void list_iterator::close() {
+ open_ = false;
+}
+
+Iterator_t list_iterator::getIterator() {
+ return this;
+}
+
+bool list_iterator::isOpen() const {
+ return open_;
+}
+
+bool list_iterator::next( Item &result ) {
+ static Item const mtime_key( factory_->createString( "mtime" ) );
+ static Item const name_key( factory_->createString( "name" ) );
+ static Item const size_key( factory_->createString( "size" ) );
+
+ string line;
+ while ( getline( is_, line ) ) {
+ if ( line.empty() )
+ continue;
+ if ( line[ line.size() - 1 ] == '\r' )
+ line.erase( line.size() - 1 );
+ struct ftpparse ftp_file;
+ if ( ftpparse( &ftp_file, line.data(), line.size() ) ) {
+ vector<pair<Item,Item> > kv;
+
+ String name( ftp_file.name, ftp_file.namelen );
+ Item const name_value( factory_->createString( name ) );
+ kv.push_back( make_pair( name_key, name_value ) );
+
+ switch ( ftp_file.sizetype ) {
+ case FTPPARSE_SIZE_ASCII:
+ case FTPPARSE_SIZE_BINARY: {
+ Item const size_value( factory_->createLong( ftp_file.size ) );
+ kv.push_back( make_pair( size_key, size_value ) );
+ break;
+ }
+ case FTPPARSE_SIZE_UNKNOWN:
+ // do nothing
+ break;
+ } // switch
+
+ struct tm tm;
+ gmtime_r( &ftp_file.mtime, &tm );
+ int const year = tm.tm_year + 1900;
+ switch ( ftp_file.mtimetype ) {
+ case FTPPARSE_MTIME_REMOTEDAY:
+ tm.tm_hour = tm.tm_min = 0;
+ // no break;
+ case FTPPARSE_MTIME_REMOTEMINUTE:
+ tm.tm_sec = 0;
+ tm.tm_gmtoff = 0;
+ // no break;
+ case FTPPARSE_MTIME_LOCAL: {
+ Item const mtime_value (
+ factory_->createDateTime(
+ year, tm.tm_mon, tm.tm_mday,
+ tm.tm_hour, tm.tm_min, tm.tm_sec, (int)tm.tm_gmtoff
+ )
+ );
+ kv.push_back( make_pair( mtime_key, mtime_value ) );
+ // no break;
+ }
+ case FTPPARSE_MTIME_UNKNOWN:
+ // do nothing
+ break;
+ } // switch
+
+ result = factory_->createJSONObject( kv );
+ return true;
+ } // if
+ } // while
+ return false;
+}
+
+void list_iterator::open() {
+ open_ = true;
+}
+
+list_function::list_function( module const *m ) :
+ function( m, "list" )
+{
+}
+
+ItemSequence_t
+list_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const conn( get_string_arg( args, 0 ) );
+ String const path( get_string_arg( args, 1 ) );
+ String const uri( make_uri( conn, path, true ) );
+
+ curl::streambuf *const cbuf = require_connection( dctx, conn );
+ CURL *const cobj = cbuf->curl();
+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_URL, uri.c_str() ) );
+ try {
+ return ItemSequence_t(
+ new list_iterator( cbuf, module_->getItemFactory() )
+ );
+ }
+ catch ( std::exception const &e ) {
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+mkdir_function::mkdir_function( module const *m ) :
+ function( m, "mkdir" )
+{
+}
+
+ItemSequence_t
+mkdir_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const conn( get_string_arg( args, 0 ) );
+ String const path( get_string_arg( args, 1 ) );
+ if ( path.empty() )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
+ String const command( "MKD " + path );
+
+ curl::streambuf *const cbuf = require_connection( dctx, conn );
+ CURL *const cobj = cbuf->curl();
+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, command.c_str() );
+
+ try {
+ curl_helper helper( cbuf );
+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
+ }
+ catch ( curl::exception const &e ) {
+ int const ftp_code = get_ftp_reply_code( cobj );
+ switch ( ftp_code ) {
+ case FTP_REPLY_ACTION_NOT_TAKEN:
+ THROW_EXCEPTION( "FTP_ERROR", path, "directory exists", ftp_code );
+ case FTP_REPLY_PATH_CREATED:
+ if ( e.curl_code() == CURLE_FTP_COULDNT_RETR_FILE ) {
+ //
+ // After the directory is created, CURL tries to RETR it which fails,
+ // so ignore that error.
+ // (Also see <http://stackoverflow.com/a/13515807/99089>.)
+ //
+ break;
+ }
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
+ default:
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what(), ftp_code );
+ } // switch
+ }
+ catch ( std::exception const &e ) {
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
+ }
+ return ItemSequence_t( new EmptySequence() );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+put_binary_function::put_binary_function( module const *m ) :
+ put_function( m, "put-binary", false )
+{
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+put_text_function::put_text_function( module const *m ) :
+ put_function( m, "put-text", true )
+{
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+static size_t curl_header_callback( void *ptr, size_t size, size_t nmemb,
+ void *data ) {
+ size *= nmemb;
+ String *const ftp_reply_msg = static_cast<String*>( data );
+
+ // skip 3-digit reply code and 1 space
+ char const *s = static_cast<char*>( ptr ) + 4;
+ size_t len = size - 4;
+
+ // trim trailing whitespace
+ while ( len && isspace( s[ len - 1 ] ) )
+ --len;
+
+ ftp_reply_msg->assign( s, len );
+ return size; // must always return original size
+}
+
+rename_function::rename_function( module const *m ) :
+ function( m, "rename" )
+{
+}
+
+ItemSequence_t
+rename_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const conn( get_string_arg( args, 0 ) );
+ String const from_path( get_string_arg( args, 1 ) );
+ if ( from_path.empty() )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "\"from\" path empty" );
+ String const to_path( get_string_arg( args, 2 ) );
+ if ( to_path.empty() )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "\to\" path empty" );
+ String const command1( "RNFR " + from_path );
+ String const command2( "RNTO " + to_path );
+
+ curl::streambuf *const cbuf = require_connection( dctx, conn );
+ CURL *const cobj = cbuf->curl();
+
+ curl_slist *slist = 0;
+ slist = curl_slist_append( slist, command1.c_str() );
+ slist = curl_slist_append( slist, command2.c_str() );
+ curl_easy_setopt( cobj, CURLOPT_QUOTE, slist );
+
+ //
+ // When doing a rename, there isn't a 1-to-1 mapping between an FTP reply
+ // code and a specific error; hence we can't use our own error message.
+ // Instead, we capture the FTP replies and parse the messages out of them.
+ // We actually only care about the last one since that's the one for the
+ // error. We then use that for the error message.
+ //
+ curl_easy_setopt( cobj, CURLOPT_HEADERFUNCTION, curl_header_callback );
+ String ftp_reply_msg;
+ curl_easy_setopt( cobj, CURLOPT_HEADERDATA, &ftp_reply_msg );
+
+ try {
+ curl_helper helper( cbuf, slist );
+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
+ }
+ catch ( curl::exception const &e ) {
+ int const ftp_code = get_ftp_reply_code( cobj );
+ switch ( ftp_code ) {
+ case FTP_REPLY_ACTION_ABORTED:
+ THROW_EXCEPTION( "FTP_ERROR", to_path, ftp_reply_msg, ftp_code );
+ case FTP_REPLY_ACTION_NOT_TAKEN:
+ THROW_EXCEPTION( "FTP_ERROR", from_path, ftp_reply_msg, ftp_code );
+ default:
+ THROW_EXCEPTION( "FTP_ERROR", from_path, e.what(), ftp_code );
+ } // switch
+ }
+ catch ( std::exception const &e ) {
+ THROW_EXCEPTION( "FTP_ERROR", from_path, e.what() );
+ }
+ return ItemSequence_t( new EmptySequence() );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+rmdir_function::rmdir_function( module const *m ) :
+ function( m, "rmdir" )
+{
+}
+
+ItemSequence_t
+rmdir_function::evaluate( ExternalFunction::Arguments_t const &args,
+ StaticContext const*,
+ DynamicContext const *dctx ) const {
+ String const conn( get_string_arg( args, 0 ) );
+ String const path( get_string_arg( args, 1 ) );
+ if ( path.empty() )
+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
+ String const command( "RMD " + path );
+
+ curl::streambuf *const cbuf = require_connection( dctx, conn );
+ CURL *const cobj = cbuf->curl();
+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, command.c_str() );
+
+ try {
+ curl_helper helper( cbuf );
+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
+ }
+ catch ( curl::exception const &e ) {
+ int const ftp_code = get_ftp_reply_code( cobj );
+ switch ( ftp_code ) {
+ case FTP_REPLY_ACTION_COMPLETED:
+ if ( e.curl_code() == CURLE_FTP_COULDNT_RETR_FILE ) {
+ //
+ // After the directory is deleted, CURL tries to RETR it which fails,
+ // so ignore that error.
+ // (Also see <http://stackoverflow.com/a/13515807/99089>.)
+ //
+ break;
+ }
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
+ case FTP_REPLY_ACTION_NOT_TAKEN:
+ THROW_EXCEPTION( "FTP_ERROR", path, "directory not found", ftp_code );
+ default:
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what(), ftp_code );
+ } // switch
+ }
+ catch ( std::exception const &e ) {
+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
+ }
+ return ItemSequence_t( new EmptySequence() );
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace ftp_client
+} // namespace zorba
+/* vim:set et sw=2 ts=2: */
=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_functions.h'
--- modules/ftp-client/ftp-client.xq.src/ftp_functions.h 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/ftp-client.xq.src/ftp_functions.h 2014-01-09 19:12:12 +0000
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2006-2013 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ZORBA_FTP_CLIENT_MODULE_FUNCTIONS_H
+#define ZORBA_FTP_CLIENT_MODULE_FUNCTIONS_H
+
+// Zorba
+#include <zorba/function.h>
+#include <zorba/item.h>
+#include <zorba/zorba_string.h>
+
+#include "curl_streambuf.h"
+
+namespace zorba {
+namespace ftp_client {
+
+class module;
+
+///////////////////////////////////////////////////////////////////////////////
+
+class function : public ContextualExternalFunction {
+public:
+ // inherited
+ virtual String getLocalName() const;
+ virtual String getURI() const;
+
+protected:
+ function( module const *m, char const *local_name );
+
+ bool get_bool_opt( Item const&, char const*, bool = false ) const;
+ int get_integer_opt( Item const&, char const*, int = 0 ) const;
+ Item get_item_arg( ExternalFunction::Arguments_t const&, unsigned ) const;
+ String get_string_arg( ExternalFunction::Arguments_t const&, unsigned ) const;
+ String get_string_opt( Item const&, char const*, char const* = "" ) const;
+
+ curl::streambuf* require_connection( DynamicContext const*,
+ String const& ) const;
+
+ void throw_exception( char const*, char const*, char const*, int = 0 ) const;
+
+ void throw_exception( char const *error_code, String const &s,
+ char const *message, int ftp_code = 0 ) const {
+ throw_exception( error_code, s.c_str(), message, ftp_code );
+ }
+
+ void throw_exception( char const *error_code, String const &s,
+ String const &message, int ftp_code = 0 ) const {
+ throw_exception( error_code, s.c_str(), message.c_str(), ftp_code );
+ }
+
+ module const *const module_;
+ char const *const local_name_; // points to C string literal
+};
+
+class get_function : public function {
+public:
+ // inherited
+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
+ StaticContext const*, DynamicContext const* ) const;
+
+protected:
+ get_function( module const*, char const *local_name, bool text );
+
+ bool const text_;
+};
+
+class put_function : public function {
+public:
+ // inherited
+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
+ StaticContext const*, DynamicContext const* ) const;
+
+protected:
+ put_function( module const*, char const *local_name, bool text );
+
+ bool const text_;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+struct connect_function : function {
+ connect_function( module const* );
+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
+ StaticContext const*, DynamicContext const* ) const;
+};
+
+struct disconnect_function : function {
+ disconnect_function( module const* );
+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
+ StaticContext const*, DynamicContext const* ) const;
+};
+
+struct delete_function : function {
+ delete_function( module const* );
+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
+ StaticContext const*, DynamicContext const* ) const;
+};
+
+struct get_binary_function : get_function {
+ get_binary_function( module const* );
+};
+
+struct get_text_function : get_function {
+ get_text_function( module const* );
+};
+
+struct list_function : function {
+ list_function( module const* );
+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
+ StaticContext const*, DynamicContext const* ) const;
+};
+
+struct mkdir_function : function {
+ mkdir_function( module const* );
+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
+ StaticContext const*, DynamicContext const* ) const;
+};
+
+struct put_binary_function : put_function {
+ put_binary_function( module const* );
+};
+
+struct put_text_function : put_function {
+ put_text_function( module const* );
+};
+
+struct rename_function : function {
+ rename_function( module const* );
+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
+ StaticContext const*, DynamicContext const* ) const;
+};
+
+struct rmdir_function : function {
+ rmdir_function( module const* );
+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
+ StaticContext const*, DynamicContext const* ) const;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace ftp_client
+} // namespace zorba
+#endif /* ZORBA_FTP_CLIENT_MODULE_FUNCTIONS_H */
+/* vim:set et sw=2 ts=2: */
=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_module.cpp'
--- modules/ftp-client/ftp-client.xq.src/ftp_module.cpp 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/ftp-client.xq.src/ftp_module.cpp 2014-01-09 19:12:12 +0000
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2006-2013 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Zorba
+#include <zorba/zorba.h>
+
+// local
+#include "ftp_functions.h"
+#include "ftp_module.h"
+
+namespace zorba {
+namespace ftp_client {
+
+///////////////////////////////////////////////////////////////////////////////
+
+module::module() {
+ item_factory_ = 0;
+}
+
+module::~module() {
+ for ( func_map_type::const_iterator i = func_map_.begin();
+ i != func_map_.end(); ++i ) {
+ delete i->second;
+ }
+}
+
+void module::destroy() {
+ delete this;
+}
+
+ExternalFunction* module::getExternalFunction( String const &local_name ) {
+ ExternalFunction *&f = func_map_[ local_name ];
+ if ( !f ) {
+ if ( local_name == "connect" )
+ f = new connect_function( this );
+ else if ( local_name == "delete" )
+ f = new delete_function( this );
+ else if ( local_name == "disconnect" )
+ f = new disconnect_function( this );
+ else if ( local_name == "get-binary" )
+ f = new get_binary_function( this );
+ else if ( local_name == "get-text" )
+ f = new get_text_function( this );
+ else if ( local_name == "list" )
+ f = new list_function( this );
+ else if ( local_name == "mkdir" )
+ f = new mkdir_function( this );
+ else if ( local_name == "put-binary" )
+ f = new put_binary_function( this );
+ else if ( local_name == "put-text" )
+ f = new put_text_function( this );
+ else if ( local_name == "rename" )
+ f = new rename_function( this );
+ else if ( local_name == "rmdir" )
+ f = new rmdir_function( this );
+ }
+ return f;
+}
+
+ItemFactory* module::getItemFactory() const {
+ if ( !item_factory_ )
+ item_factory_ = Zorba::getInstance(0)->getItemFactory();
+ return item_factory_;
+}
+
+String module::getURI() const {
+ static String const uri( "http://zorba.io/modules/ftp-client"; );
+ return uri;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace ftp_client
+} // namespace zorba
+
+#ifdef WIN32
+# define DLL_EXPORT __declspec(dllexport)
+#else
+# define DLL_EXPORT __attribute__ ((visibility("default")))
+#endif /* WIN32 */
+
+extern "C" DLL_EXPORT zorba::ExternalModule* createModule() {
+ return new zorba::ftp_client::module();
+}
+
+/* vim:set et sw=2 ts=2: */
=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_module.h'
--- modules/ftp-client/ftp-client.xq.src/ftp_module.h 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/ftp-client.xq.src/ftp_module.h 2014-01-09 19:12:12 +0000
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2006-2013 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ZORBA_FTP_CLIENT_FTP_MODULE_H
+#define ZORBA_FTP_CLIENT_FTP_MODULE_H
+
+// standard
+#include <map>
+
+// Zorba
+#include <zorba/external_module.h>
+
+namespace zorba {
+namespace ftp_client {
+
+///////////////////////////////////////////////////////////////////////////////
+
+class module : public ExternalModule {
+public:
+ module();
+ ~module();
+
+ ItemFactory* getItemFactory() const;
+
+ // inherited
+ virtual void destroy();
+ virtual ExternalFunction* getExternalFunction( String const& );
+ virtual String getURI() const;
+
+private:
+ // map function names -> ExternalFunction*
+ typedef std::map<String,ExternalFunction*> func_map_type;
+ mutable func_map_type func_map_;
+ mutable ItemFactory *item_factory_;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace ftp_client
+} // namespace zorba
+
+#endif /* ZORBA_FTP_CLIENT_FTP_MODULE_H */
+/* vim:set et sw=2 ts=2: */
=== added file 'modules/ftp-client/ftp-client.xq.src/ftpparse.c'
--- modules/ftp-client/ftp-client.xq.src/ftpparse.c 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/ftp-client.xq.src/ftpparse.c 2014-01-09 19:12:12 +0000
@@ -0,0 +1,448 @@
+/* ftpparse.c, ftpparse.h: library for parsing FTP LIST responses
+20001223
+D. J. Bernstein, djb@xxxxxxxx
+http://cr.yp.to/ftpparse.html
+
+Commercial use is fine, if you let me know what programs you're using this in.
+
+Currently covered formats:
+EPLF.
+UNIX ls, with or without gid.
+Microsoft FTP Service.
+Windows NT FTP Server.
+VMS.
+WFTPD.
+NetPresenz (Mac).
+NetWare.
+MSDOS.
+
+Definitely not covered:
+Long VMS filenames, with information split across two lines.
+NCSA Telnet FTP server. Has LIST = NLST (and bad NLST for directories).
+*/
+
+#include <time.h>
+#include "ftpparse.h"
+
+static long totai(long year,long month,long mday)
+{
+ long result;
+ if (month >= 2) month -= 2;
+ else { month += 10; --year; }
+ result = (mday - 1) * 10 + 5 + 306 * month;
+ result /= 10;
+ if (result == 365) { year -= 3; result = 1460; }
+ else result += 365 * (year % 4);
+ year /= 4;
+ result += 1461 * (year % 25);
+ year /= 25;
+ if (result == 36524) { year -= 3; result = 146096; }
+ else { result += 36524 * (year % 4); }
+ year /= 4;
+ result += 146097 * (year - 5);
+ result += 11017;
+ return result * 86400;
+}
+
+static int flagneedbase = 1;
+static time_t base; /* time() value on this OS at the beginning of 1970 TAI */
+static long now; /* current time */
+static int flagneedcurrentyear = 1;
+static long currentyear; /* approximation to current year */
+
+static void initbase(void)
+{
+ struct tm *t;
+ if (!flagneedbase) return;
+
+ base = 0;
+ t = gmtime(&base);
+ base = -(totai(t->tm_year + 1900,t->tm_mon,t->tm_mday) + t->tm_hour * 3600 + t->tm_min * 60 + t->tm_sec);
+ /* assumes the right time_t, counting seconds. */
+ /* base may be slightly off if time_t counts non-leap seconds. */
+ flagneedbase = 0;
+}
+
+static void initnow(void)
+{
+ long day;
+ long year;
+
+ initbase();
+ now = time((time_t *) 0) - base;
+
+ if (flagneedcurrentyear) {
+ day = now / 86400;
+ if ((now % 86400) < 0) --day;
+ day -= 11017;
+ year = 5 + day / 146097;
+ day = day % 146097;
+ if (day < 0) { day += 146097; --year; }
+ year *= 4;
+ if (day == 146096) { year += 3; day = 36524; }
+ else { year += day / 36524; day %= 36524; }
+ year *= 25;
+ year += day / 1461;
+ day %= 1461;
+ year *= 4;
+ if (day == 1460) { year += 3; day = 365; }
+ else { year += day / 365; day %= 365; }
+ day *= 10;
+ if ((day + 5) / 306 >= 10) ++year;
+ currentyear = year;
+ flagneedcurrentyear = 0;
+ }
+}
+
+/* UNIX ls does not show the year for dates in the last six months. */
+/* So we have to guess the year. */
+/* Apparently NetWare uses ``twelve months'' instead of ``six months''; ugh. */
+/* Some versions of ls also fail to show the year for future dates. */
+static long guesstai(long month,long mday)
+{
+ long year;
+ long t;
+
+ initnow();
+
+ for (year = currentyear - 1;year < currentyear + 100;++year) {
+ t = totai(year,month,mday);
+ if (now - t < 350 * 86400)
+ return t;
+ }
+ return 0;
+}
+
+static int check(const char *buf,const char *monthname)
+{
+ if ((buf[0] != monthname[0]) && (buf[0] != monthname[0] - 32)) return 0;
+ if ((buf[1] != monthname[1]) && (buf[1] != monthname[1] - 32)) return 0;
+ if ((buf[2] != monthname[2]) && (buf[2] != monthname[2] - 32)) return 0;
+ return 1;
+}
+
+static const char *months[12] = {
+ "jan","feb","mar","apr","may","jun","jul","aug","sep","oct","nov","dec"
+} ;
+
+static int getmonth(const char *buf,int len)
+{
+ int i;
+ if (len == 3)
+ for (i = 0;i < 12;++i)
+ if (check(buf,months[i])) return i;
+ return -1;
+}
+
+static long getlong(const char *buf,int len)
+{
+ long u = 0;
+ while (len-- > 0)
+ u = u * 10 + (*buf++ - '0');
+ return u;
+}
+
+int ftpparse(struct ftpparse *fp,const char *buf,int len)
+{
+ int i;
+ int j;
+ int state;
+ long size;
+ long year;
+ long month;
+ long mday;
+ long hour;
+ long minute;
+
+ fp->name = 0;
+ fp->namelen = 0;
+ fp->flagtrycwd = 0;
+ fp->flagtryretr = 0;
+ fp->sizetype = FTPPARSE_SIZE_UNKNOWN;
+ fp->size = 0;
+ fp->mtimetype = FTPPARSE_MTIME_UNKNOWN;
+ fp->mtime = 0;
+ fp->idtype = FTPPARSE_ID_UNKNOWN;
+ fp->id = 0;
+ fp->idlen = 0;
+
+ if (len < 2) /* an empty name in EPLF, with no info, could be 2 chars */
+ return 0;
+
+ switch(*buf) {
+ /* see http://pobox.com/~djb/proto/eplf.txt */
+ /* "+i8388621.29609,m824255902,/,\tdev" */
+ /* "+i8388621.44468,m839956783,r,s10376,\tRFCEPLF" */
+ case '+':
+ i = 1;
+ for (j = 1;j < len;++j) {
+ if (buf[j] == 9) {
+ fp->name = buf + j + 1;
+ fp->namelen = len - j - 1;
+ return 1;
+ }
+ if (buf[j] == ',') {
+ switch(buf[i]) {
+ case '/':
+ fp->flagtrycwd = 1;
+ break;
+ case 'r':
+ fp->flagtryretr = 1;
+ break;
+ case 's':
+ fp->sizetype = FTPPARSE_SIZE_BINARY;
+ fp->size = getlong(buf + i + 1,j - i - 1);
+ break;
+ case 'm':
+ fp->mtimetype = FTPPARSE_MTIME_LOCAL;
+ initbase();
+ fp->mtime = base + getlong(buf + i + 1,j - i - 1);
+ break;
+ case 'i':
+ fp->idtype = FTPPARSE_ID_FULL;
+ fp->id = buf + i + 1;
+ fp->idlen = j - i - 1;
+ }
+ i = j + 1;
+ }
+ }
+ return 0;
+
+ /* UNIX-style listing, without inum and without blocks */
+ /* "-rw-r--r-- 1 root other 531 Jan 29 03:26 README" */
+ /* "dr-xr-xr-x 2 root other 512 Apr 8 1994 etc" */
+ /* "dr-xr-xr-x 2 root 512 Apr 8 1994 etc" */
+ /* "lrwxrwxrwx 1 root other 7 Jan 25 00:17 bin -> usr/bin" */
+ /* Also produced by Microsoft's FTP servers for Windows: */
+ /* "---------- 1 owner group 1803128 Jul 10 10:18 ls-lR.Z" */
+ /* "d--------- 1 owner group 0 May 9 19:45 Softlib" */
+ /* Also WFTPD for MSDOS: */
+ /* "-rwxrwxrwx 1 noone nogroup 322 Aug 19 1996 message.ftp" */
+ /* Also NetWare: */
+ /* "d [R----F--] supervisor 512 Jan 16 18:53 login" */
+ /* "- [R----F--] rhesus 214059 Oct 20 15:27 cx.exe" */
+ /* Also NetPresenz for the Mac: */
+ /* "-------r-- 326 1391972 1392298 Nov 22 1995 MegaPhone.sit" */
+ /* "drwxrwxr-x folder 2 May 10 1996 network" */
+ case 'b':
+ case 'c':
+ case 'd':
+ case 'l':
+ case 'p':
+ case 's':
+ case '-':
+
+ if (*buf == 'd') fp->flagtrycwd = 1;
+ if (*buf == '-') fp->flagtryretr = 1;
+ if (*buf == 'l') fp->flagtrycwd = fp->flagtryretr = 1;
+
+ state = 1;
+ i = 0;
+ for (j = 1;j < len;++j)
+ if ((buf[j] == ' ') && (buf[j - 1] != ' ')) {
+ switch(state) {
+ case 1: /* skipping perm */
+ state = 2;
+ break;
+ case 2: /* skipping nlink */
+ state = 3;
+ if ((j - i == 6) && (buf[i] == 'f')) /* for NetPresenz */
+ state = 4;
+ break;
+ case 3: /* skipping uid */
+ state = 4;
+ break;
+ case 4: /* getting tentative size */
+ size = getlong(buf + i,j - i);
+ state = 5;
+ break;
+ case 5: /* searching for month, otherwise getting tentative size */
+ month = getmonth(buf + i,j - i);
+ if (month >= 0)
+ state = 6;
+ else
+ size = getlong(buf + i,j - i);
+ break;
+ case 6: /* have size and month */
+ mday = getlong(buf + i,j - i);
+ state = 7;
+ break;
+ case 7: /* have size, month, mday */
+ if ((j - i == 4) && (buf[i + 1] == ':')) {
+ hour = getlong(buf + i,1);
+ minute = getlong(buf + i + 2,2);
+ fp->mtimetype = FTPPARSE_MTIME_REMOTEMINUTE;
+ initbase();
+ fp->mtime = base + guesstai(month,mday) + hour * 3600 + minute * 60;
+ } else if ((j - i == 5) && (buf[i + 2] == ':')) {
+ hour = getlong(buf + i,2);
+ minute = getlong(buf + i + 3,2);
+ fp->mtimetype = FTPPARSE_MTIME_REMOTEMINUTE;
+ initbase();
+ fp->mtime = base + guesstai(month,mday) + hour * 3600 + minute * 60;
+ }
+ else if (j - i >= 4) {
+ year = getlong(buf + i,j - i);
+ fp->mtimetype = FTPPARSE_MTIME_REMOTEDAY;
+ initbase();
+ fp->mtime = base + totai(year,month,mday);
+ }
+ else
+ return 0;
+ fp->name = buf + j + 1;
+ fp->namelen = len - j - 1;
+ state = 8;
+ break;
+ case 8: /* twiddling thumbs */
+ break;
+ }
+ i = j + 1;
+ while ((i < len) && (buf[i] == ' ')) ++i;
+ }
+
+ if (state != 8)
+ return 0;
+
+ fp->size = size;
+ fp->sizetype = FTPPARSE_SIZE_BINARY;
+
+ if (*buf == 'l')
+ for (i = 0;i + 3 < fp->namelen;++i)
+ if (fp->name[i] == ' ')
+ if (fp->name[i + 1] == '-')
+ if (fp->name[i + 2] == '>')
+ if (fp->name[i + 3] == ' ') {
+ fp->namelen = i;
+ break;
+ }
+
+ /* eliminate extra NetWare spaces */
+ if ((buf[1] == ' ') || (buf[1] == '['))
+ if (fp->namelen > 3)
+ if (fp->name[0] == ' ')
+ if (fp->name[1] == ' ')
+ if (fp->name[2] == ' ') {
+ fp->name += 3;
+ fp->namelen -= 3;
+ }
+
+ return 1;
+ }
+
+ /* MultiNet (some spaces removed from examples) */
+ /* "00README.TXT;1 2 30-DEC-1996 17:44 [SYSTEM] (RWED,RWED,RE,RE)" */
+ /* "CORE.DIR;1 1 8-SEP-1996 16:09 [SYSTEM] (RWE,RWE,RE,RE)" */
+ /* and non-MutliNet VMS: */
+ /* "CII-MANUAL.TEX;1 213/216 29-JAN-1996 03:33:12 [ANONYMOU,ANONYMOUS] (RWED,RWED,,)" */
+ for (i = 0;i < len;++i)
+ if (buf[i] == ';')
+ break;
+ if (i < len) {
+ fp->name = buf;
+ fp->namelen = i;
+ if (i > 4)
+ if (buf[i - 4] == '.')
+ if (buf[i - 3] == 'D')
+ if (buf[i - 2] == 'I')
+ if (buf[i - 1] == 'R') {
+ fp->namelen -= 4;
+ fp->flagtrycwd = 1;
+ }
+ if (!fp->flagtrycwd)
+ fp->flagtryretr = 1;
+ while (buf[i] != ' ') if (++i == len) return 0;
+ while (buf[i] == ' ') if (++i == len) return 0;
+ while (buf[i] != ' ') if (++i == len) return 0;
+ while (buf[i] == ' ') if (++i == len) return 0;
+ j = i;
+ while (buf[j] != '-') if (++j == len) return 0;
+ mday = getlong(buf + i,j - i);
+ while (buf[j] == '-') if (++j == len) return 0;
+ i = j;
+ while (buf[j] != '-') if (++j == len) return 0;
+ month = getmonth(buf + i,j - i);
+ if (month < 0) return 0;
+ while (buf[j] == '-') if (++j == len) return 0;
+ i = j;
+ while (buf[j] != ' ') if (++j == len) return 0;
+ year = getlong(buf + i,j - i);
+ while (buf[j] == ' ') if (++j == len) return 0;
+ i = j;
+ while (buf[j] != ':') if (++j == len) return 0;
+ hour = getlong(buf + i,j - i);
+ while (buf[j] == ':') if (++j == len) return 0;
+ i = j;
+ while ((buf[j] != ':') && (buf[j] != ' ')) if (++j == len) return 0;
+ minute = getlong(buf + i,j - i);
+
+ fp->mtimetype = FTPPARSE_MTIME_REMOTEMINUTE;
+ initbase();
+ fp->mtime = base + totai(year,month,mday) + hour * 3600 + minute * 60;
+
+ return 1;
+ }
+
+ /* MSDOS format */
+ /* 04-27-00 09:09PM <DIR> licensed */
+ /* 07-18-00 10:16AM <DIR> pub */
+ /* 04-14-00 03:47PM 589 readme.htm */
+ if ((*buf >= '0') && (*buf <= '9')) {
+ i = 0;
+ j = 0;
+ while (buf[j] != '-') if (++j == len) return 0;
+ month = getlong(buf + i,j - i) - 1;
+ while (buf[j] == '-') if (++j == len) return 0;
+ i = j;
+ while (buf[j] != '-') if (++j == len) return 0;
+ mday = getlong(buf + i,j - i);
+ while (buf[j] == '-') if (++j == len) return 0;
+ i = j;
+ while (buf[j] != ' ') if (++j == len) return 0;
+ year = getlong(buf + i,j - i);
+ if (year < 50) year += 2000;
+ if (year < 1000) year += 1900;
+ while (buf[j] == ' ') if (++j == len) return 0;
+ i = j;
+ while (buf[j] != ':') if (++j == len) return 0;
+ hour = getlong(buf + i,j - i);
+ while (buf[j] == ':') if (++j == len) return 0;
+ i = j;
+ while ((buf[j] != 'A') && (buf[j] != 'P')) if (++j == len) return 0;
+ minute = getlong(buf + i,j - i);
+ if (hour == 12) hour = 0;
+ if (buf[j] == 'A') if (++j == len) return 0;
+ if (buf[j] == 'P') { hour += 12; if (++j == len) return 0; }
+ if (buf[j] == 'M') if (++j == len) return 0;
+
+ while (buf[j] == ' ') if (++j == len) return 0;
+ if (buf[j] == '<') {
+ fp->flagtrycwd = 1;
+ while (buf[j] != ' ') if (++j == len) return 0;
+ }
+ else {
+ i = j;
+ while (buf[j] != ' ') if (++j == len) return 0;
+ fp->size = getlong(buf + i,j - i);
+ fp->sizetype = FTPPARSE_SIZE_BINARY;
+ fp->flagtryretr = 1;
+ }
+ while (buf[j] == ' ') if (++j == len) return 0;
+
+ fp->name = buf + j;
+ fp->namelen = len - j;
+
+ fp->mtimetype = FTPPARSE_MTIME_REMOTEMINUTE;
+ initbase();
+ fp->mtime = base + totai(year,month,mday) + hour * 3600 + minute * 60;
+
+ return 1;
+ }
+
+ /* Some useless lines, safely ignored: */
+ /* "Total of 11 Files, 10966 Blocks." (VMS) */
+ /* "total 14786" (UNIX) */
+ /* "DISK$ANONFTP:[ANONYMOUS]" (VMS) */
+ /* "Directory DISK$PCSA:[ANONYM]" (VMS) */
+
+ return 0;
+}
=== added file 'modules/ftp-client/ftp-client.xq.src/ftpparse.h'
--- modules/ftp-client/ftp-client.xq.src/ftpparse.h 1970-01-01 00:00:00 +0000
+++ modules/ftp-client/ftp-client.xq.src/ftpparse.h 2014-01-09 19:12:12 +0000
@@ -0,0 +1,51 @@
+#ifndef FTPPARSE_H
+#define FTPPARSE_H
+
+/*
+ftpparse(&fp,buf,len) tries to parse one line of LIST output.
+
+The line is an array of len characters stored in buf.
+It should not include the terminating CR LF; so buf[len] is typically CR.
+
+If ftpparse() can't find a filename, it returns 0.
+
+If ftpparse() can find a filename, it fills in fp and returns 1.
+fp is a struct ftpparse, defined below.
+The name is an array of fp.namelen characters stored in fp.name;
+fp.name points somewhere within buf.
+*/
+
+struct ftpparse {
+ const char *name; /* not necessarily 0-terminated */
+ int namelen;
+ int flagtrycwd; /* 0 if cwd is definitely pointless, 1 otherwise */
+ int flagtryretr; /* 0 if retr is definitely pointless, 1 otherwise */
+ int sizetype;
+ long size; /* number of octets */
+ int mtimetype;
+ time_t mtime; /* modification time */
+ int idtype;
+ const char *id; /* not necessarily 0-terminated */
+ int idlen;
+} ;
+
+#define FTPPARSE_SIZE_UNKNOWN 0
+#define FTPPARSE_SIZE_BINARY 1 /* size is the number of octets in TYPE I */
+#define FTPPARSE_SIZE_ASCII 2 /* size is the number of octets in TYPE A */
+
+#define FTPPARSE_MTIME_UNKNOWN 0
+#define FTPPARSE_MTIME_LOCAL 1 /* time is correct */
+#define FTPPARSE_MTIME_REMOTEMINUTE 2 /* time zone and secs are unknown */
+#define FTPPARSE_MTIME_REMOTEDAY 3 /* time zone and time of day are unknown */
+/*
+When a time zone is unknown, it is assumed to be GMT. You may want
+to use localtime() for LOCAL times, along with an indication that the
+time is correct in the local time zone, and gmtime() for REMOTE* times.
+*/
+
+#define FTPPARSE_ID_UNKNOWN 0
+#define FTPPARSE_ID_FULL 1 /* unique identifier for files on this FTP server */
+
+extern int ftpparse(struct ftpparse *,const char *,int);
+
+#endif
=== modified file 'modules/http-client/CMakeLists.txt'
--- modules/http-client/CMakeLists.txt 2013-07-25 14:47:04 +0000
+++ modules/http-client/CMakeLists.txt 2014-01-09 19:12:12 +0000
@@ -1,64 +1,52 @@
# Copyright 2006-2012 The FLWOR Foundation.
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
-# cURL
-#
-SET (CURL_FOUND)
-IF(ZORBA_SUPPRESS_CURL)
- MESSAGE(STATUS "ZORBA_SUPPRESS_CURL is true - not searching for cURL library")
-ELSE(ZORBA_SUPPRESS_CURL)
- MESSAGE(STATUS "Looking for cURL")
- FIND_PACKAGE(CURL)
-
- IF(CURL_FOUND)
- MESSAGE(STATUS "Found cURL library -- " ${CURL_LIBRARIES})
- INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
+IF (ZORBA_HAVE_CURL)
+ SET(ZorbaUtilCurlModule_DIR "../util-curl")
+ FIND_PACKAGE(ZorbaUtilCurlModule REQUIRED)
+ IF (ZorbaUtilCurlModule_FOUND)
+ INCLUDE_DIRECTORIES("${ZorbaUtilCurlModule_INCLUDE_DIRS}")
DECLARE_ZORBA_MODULE(FILE xml/http-client-error.xq
URI "http://expath.org/ns/error";)
-
+
DECLARE_ZORBA_SCHEMA(FILE xml/http-client.xsd
URI "http://expath.org/ns/http-client";)
-
+
DECLARE_ZORBA_MODULE(FILE xml/http-client.xq VERSION 2.0
URI "http://www.zorba-xquery.com/modules/http-client";
- LINK_LIBRARIES ${CURL_LIBRARIES})
-
+ LINK_LIBRARIES ${CURL_LIBRARIES} ${ZorbaUtilCurlModule_LIBS})
+
DECLARE_ZORBA_MODULE(FILE json/http-client.xq VERSION 1.0
URI "http://zorba.io/modules/http-client";
- LINK_LIBRARIES ${CURL_LIBRARIES})
-
+ LINK_LIBRARIES ${CURL_LIBRARIES} ${ZorbaUtilCurlModule_LIBS})
+
DECLARE_ZORBA_MODULE(FILE conv/http-client-wrapper.xq VERSION 1.0
URI "http://zorba.io/modules/http-client-wrapper";
- LINK_LIBRARIES ${CURL_LIBRARIES})
-
+ LINK_LIBRARIES ${CURL_LIBRARIES} ${ZorbaUtilCurlModule_LIBS})
+
IF (WIN32) # Copy certificates for windows only
IF (MSVC_IDE)
SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/${CMAKE_BUILD_TYPE}/cacert.pem")
ELSE (MSVC_IDE)
- SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/cacert.pem")
+ SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/cacert.pem")
ENDIF (MSVC_IDE)
CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/cacert.pem" ${CACERT_DESTINATION} COPYONLY)
INSTALL(FILES ${CACERT_DESTINATION} DESTINATION bin)
- ENDIF (WIN32)
-
- ELSE(CURL_FOUND)
- MESSAGE(STATUS "The cURL library was not found - http-client will not be built")
- ENDIF(CURL_FOUND)
-ENDIF(ZORBA_SUPPRESS_CURL)
-SET(ZORBA_HAVE_CURL ${CURL_FOUND} CACHE BOOL "Whether Zorba found cURL" FORCE)
-MARK_AS_ADVANCED(ZORBA_HAVE_CURL)
+ ENDIF (WIN32)
+
+ ENDIF (ZorbaUtilCurlModule_FOUND)
+ENDIF (ZORBA_HAVE_CURL)
# vim:set et sw=2 ts=2:
=== modified file 'modules/http-client/json/http-client.xq'
--- modules/http-client/json/http-client.xq 2013-09-26 23:15:11 +0000
+++ modules/http-client/json/http-client.xq 2014-01-09 19:12:12 +0000
@@ -186,6 +186,7 @@
module namespace http = "http://zorba.io/modules/http-client";;
import module namespace libjn = "http://jsoniq.org/function-library";;
+import module namespace util-curl = "http://zorba.io/modules/util-curl";;
declare namespace an = "http://zorba.io/annotations";;
declare namespace ver = "http://zorba.io/options/versioning";;
=== removed file 'modules/http-client/json/http-client.xq.src/curl_stream_buffer.cpp'
--- modules/http-client/json/http-client.xq.src/curl_stream_buffer.cpp 2013-06-14 16:32:01 +0000
+++ modules/http-client/json/http-client.xq.src/curl_stream_buffer.cpp 1970-01-01 00:00:00 +0000
@@ -1,379 +0,0 @@
-/*
- * Copyright 2006-2013 The FLWOR Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <zorba/config.h>
-
-#include <cstdlib>
-#include <cstring> /* for memcpy(3) */
-#include <iostream>
-#include <cassert>
-#ifndef WIN32
-#include <cerrno>
-#include <sys/time.h>
-#endif /* WIN32 */
-
-#include <curl/multi.h>
-
-#include "curl_stream_buffer.h"
-#include "inform_data_read.h"
-
-using namespace std;
-
-namespace zorba {
-namespace curl {
-
-///////////////////////////////////////////////////////////////////////////////
-
-#define ZORBA_CURL_ASSERT(expr) \
- do { \
- if ( CURLcode const code##__LINE__ = (expr) ) \
- throw exception( #expr, "", code##__LINE__ ); \
- } while (0)
-
-#define ZORBA_CURLM_ASSERT(expr) \
- do { \
- if ( CURLMcode const code##__LINE__ = (expr) ) \
- if ( code##__LINE__ != CURLM_CALL_MULTI_PERFORM ) \
- throw exception( #expr, "", code##__LINE__ ); \
- } while (0)
-
-exception::exception( char const *function, char const *uri, char const *msg ) :
- std::exception(), msg_( msg )
-{
-}
-
-exception::exception( char const *function, char const *uri, CURLcode code ) :
- std::exception(),
- msg_( curl_easy_strerror( code ) )
-{
-}
-
-exception::exception( char const *function, char const *uri, CURLMcode code ) :
- std::exception(),
- msg_( curl_multi_strerror( code ) )
-{
-}
-
-exception::~exception() throw() {
- // out-of-line since it's virtual
-}
-
-const char* exception::what() const throw() {
- return msg_.c_str();
-}
-
-///////////////////////////////////////////////////////////////////////////////
-
-CURL* create( char const *uri, write_fn_t fn, void *data ) {
- //
- // Having cURL initialization wrapped by a class and using a singleton static
- // instance guarantees that cURL is initialized exactly once before use and
- // and also is cleaned-up at program termination (when destructors for static
- // objects are called).
- //
- struct curl_initializer {
- curl_initializer() {
- ZORBA_CURL_ASSERT( curl_global_init( CURL_GLOBAL_ALL ) );
- }
- ~curl_initializer() {
- curl_global_cleanup();
- }
- };
- static curl_initializer initializer;
-
- CURL *const curl = curl_easy_init();
- if ( !curl )
- throw exception( "curl_easy_init()", uri, "" );
-
- try {
- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_URL, uri ) );
- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_WRITEDATA, data ) );
- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, fn ) );
-
- // Tells cURL to follow redirects. CURLOPT_MAXREDIRS is by default set to -1
- // thus cURL will do an infinite number of redirects.
- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_FOLLOWLOCATION, 1 ) );
-
-#ifndef ZORBA_VERIFY_PEER_SSL_CERTIFICATE
- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_SSL_VERIFYPEER, 0 ) );
- //
- // CURLOPT_SSL_VERIFYHOST is left default, value 2, meaning verify that the
- // Common Name or Subject Alternate Name field in the certificate matches
- // the name of the server.
- //
- // Tested with https://www.npr.org/rss/rss.php?id=1001
- // About using SSL certs in curl: http://curl.haxx.se/docs/sslcerts.html
-#else
-# ifdef WIN32
- // set the root CA certificates file path
- if ( GENV.g_curl_root_CA_certificates_path[0] )
- ZORBA_CURL_ASSERT(
- curl_easy_setopt(
- curl, CURLOPT_CAINFO, GENV.g_curl_root_CA_certificates_path
- )
- );
-# endif /* WIN32 */
-#endif /* ZORBA_VERIFY_PEER_SSL_CERTIFICATE */
-
- //
- // Some servers don't like requests that are made without a user-agent
- // field, so we provide one.
- //
- //ZORBA_CURL_ASSERT(
- //curl_easy_setopt( curl, CURLOPT_USERAGENT, "libcurl-agent/1.0" )
- //);
-
- return curl;
- }
- catch ( ... ) {
- destroy( curl );
- throw;
- }
-}
-
-void destroy( CURL *curl ) {
- if ( curl ) {
- curl_easy_reset( curl );
- curl_easy_cleanup( curl );
- }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-
-streambuf::streambuf() {
- init();
-}
-
-streambuf::streambuf( char const *uri ) {
- init();
- open( uri );
-}
-
-streambuf::streambuf( CURL *curl ) {
- init();
- curl_ = curl;
- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_WRITEDATA, this ) );
- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, curl_write_callback ) );
- init_curlm();
-}
-
-streambuf::~streambuf() {
- free( buf_ );
- close();
-#ifdef WIN32
- closesocket( dummy_socket_ );
-#endif
- // If we have been assigned memory ownership of theInformer, delete it now.
- if ( theOwnInformer )
- delete theInformer;
-}
-
-void streambuf::close() {
- if ( curl_ ) {
- if ( curlm_ ) {
- curl_multi_remove_handle( curlm_, curl_ );
- curl_multi_cleanup( curlm_ );
- curlm_ = 0;
- }
- destroy( curl_ );
- curl_ = 0;
- }
-}
-
-void streambuf::curl_read() {
- buf_len_ = 0;
- while ( curl_running_ && !buf_len_ ) {
- fd_set fd_read, fd_write, fd_except;
- FD_ZERO( &fd_read );
- FD_ZERO( &fd_write );
- FD_ZERO( &fd_except );
- int max_fd = -1;
-#ifdef WIN32
- //
- // Windows does not like a call to select where all arguments are 0, so we
- // just add a dummy socket to make the call to select happy.
- //
- FD_SET( dummy_socket_, &fd_read );
-#endif /* WIN32 */
- ZORBA_CURLM_ASSERT(
- curl_multi_fdset( curlm_, &fd_read, &fd_write, &fd_except, &max_fd )
- );
-
- //
- // Note that the fopen.c sample code is unnecessary at best or wrong at
- // worst; see: http://curl.haxx.se/mail/lib-2011-05/0011.html
- //
- timeval timeout;
- long curl_timeout_ms;
- ZORBA_CURLM_ASSERT( curl_multi_timeout( curlm_, &curl_timeout_ms ) );
- if ( curl_timeout_ms > 0 ) {
- timeout.tv_sec = curl_timeout_ms / 1000;
- timeout.tv_usec = curl_timeout_ms % 1000 * 1000;
- } else {
- //
- // From curl_multi_timeout(3):
- //
- // Note: if libcurl returns a -1 timeout here, it just means that
- // libcurl currently has no stored timeout value. You must not wait
- // too long (more than a few seconds perhaps) before you call
- // curl_multi_perform() again.
- //
- // So we just pick some not-too-long default.
- //
- timeout.tv_sec = 1;
- timeout.tv_usec = 0;
- }
-
- switch ( select( max_fd + 1, &fd_read, &fd_write, &fd_except, &timeout ) ) {
- case -1: // select error
-#ifdef WIN32
- char err_buf[8];
- sprintf( err_buf, "%d", WSAGetLastError() );
- throw exception( "select()", "", err_buf );
-#else
- throw exception( "select()", "", strerror( errno ) );
-#endif
- case 0: // timeout
- // no break;
- default:
- CURLMcode code;
- do {
- code = curl_multi_perform( curlm_, &curl_running_ );
- } while ( code == CURLM_CALL_MULTI_PERFORM );
- ZORBA_CURLM_ASSERT( code );
- }
- }
- if ( theInformer )
- theInformer->afterRead();
-}
-
-size_t streambuf::curl_write_callback( void *ptr, size_t size, size_t nmemb,
- void *data ) {
- size *= nmemb;
- streambuf *const that = static_cast<streambuf*>( data );
-
- if ( that->theInformer )
- that->theInformer->beforeRead();
-
- size_t const buf_free = that->buf_capacity_ - that->buf_len_;
- if ( size > buf_free ) {
- streamoff new_capacity = that->buf_capacity_ + size - buf_free;
- if ( void *const new_buf =
- realloc( that->buf_, static_cast<size_t>( new_capacity ) ) ) {
- that->buf_ = static_cast<char*>( new_buf );
- that->buf_capacity_ = new_capacity;
- } else
- throw exception( "realloc()", "" );
- }
- ::memcpy( that->buf_ + that->buf_len_, ptr, size );
- that->buf_len_ += size;
- return size;
-}
-
-void streambuf::init() {
- buf_ = 0;
- buf_capacity_ = 0;
- buf_len_ = 0;
- curl_ = 0;
- curlm_ = 0;
- curl_running_ = 0;
- theInformer = 0;
- theOwnInformer = false;
-#ifdef WIN32
- dummy_socket_ = socket( AF_INET, SOCK_DGRAM, 0 );
- if ( dummy_socket_ == CURL_SOCKET_BAD || dummy_socket_ == INVALID_SOCKET )
- throw exception( "socket()", "" );
-#endif /* WIN32 */
-}
-
-void streambuf::init_curlm() {
- //
- // Lie about cURL running initially so the while-loop in curl_read() will run
- // at least once.
- //
- curl_running_ = 1;
-
- //
- // Set the "get" pointer to the end (gptr() == egptr()) so a call to
- // underflow() and initial data read will be triggered.
- //
- buf_len_ = buf_capacity_;
- setg( buf_, buf_ + buf_len_, buf_ + buf_capacity_ );
-
- //
- // Clean-up has to be done here with try/catch (as opposed to relying on the
- // destructor) because open() can be called from the constructor. If an
- // exception is thrown, the constructor will not have completed, hence the
- // object will not have been fully constructed; therefore the destructor will
- // not be called.
- //
- try {
- if ( !(curlm_ = curl_multi_init()) )
- throw exception( "curl_multi_init()", "" );
- try {
- ZORBA_CURLM_ASSERT( curl_multi_add_handle( curlm_, curl_ ) );
- }
- catch ( ... ) {
- curl_multi_cleanup( curlm_ );
- curlm_ = 0;
- throw;
- }
- }
- catch ( ... ) {
- destroy( curl_ );
- curl_ = 0;
- throw;
- }
-}
-
-int streambuf::multi_perform() {
- underflow();
- CURLMsg *msg;
- int msgInQueue;
- int error = 0;
- while ( (msg = curl_multi_info_read( curlm_, &msgInQueue )) ) {
- if ( msg->msg == CURLMSG_DONE )
- error = msg->data.result;
- }
- return error;
-}
-
-void streambuf::open( char const *uri ) {
- curl_ = create( uri, curl_write_callback, this );
-
- init_curlm();
-}
-
-streamsize streambuf::showmanyc() {
- return egptr() - gptr();
-}
-
-streambuf::int_type streambuf::underflow() {
- while ( true ) {
- if ( gptr() < egptr() )
- return traits_type::to_int_type( *gptr() );
- curl_read();
- if ( !buf_len_ )
- return traits_type::eof();
- setg( buf_, buf_, buf_ + buf_len_ );
- }
-}
-
-///////////////////////////////////////////////////////////////////////////////
-
-} // namespace curl
-} // namespace zorba
-/* vim:set et sw=2 ts=2: */
=== removed file 'modules/http-client/json/http-client.xq.src/curl_stream_buffer.h'
--- modules/http-client/json/http-client.xq.src/curl_stream_buffer.h 2013-06-14 16:32:01 +0000
+++ modules/http-client/json/http-client.xq.src/curl_stream_buffer.h 1970-01-01 00:00:00 +0000
@@ -1,193 +0,0 @@
-/*
- * Copyright 2006-2013 The FLWOR Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#pragma once
-#ifndef ZORBA_CURL_UTIL_H
-#define ZORBA_CURL_UTIL_H
-
-#include <zorba/config.h>
-
-#include <exception>
-#include <istream>
-#include <streambuf>
-#include <string>
-#include <curl/curl.h>
-
-namespace zorba {
-
-namespace http_client {
- class InformDataRead;
-}
-
-namespace curl {
-
-///////////////////////////////////////////////////////////////////////////////
-
-class exception : public std::exception {
-public:
- exception( char const *function, char const *uri, char const *msg = 0 );
- exception( char const *function, char const *uri, CURLcode code );
- exception( char const *function, char const *uri, CURLMcode code );
- ~exception() throw();
-
- virtual const char* what() const throw();
-
-private:
- std::string msg_;
-};
-
-////////// create & destroy ///////////////////////////////////////////////////
-
-/**
- * The signature type of cURL's write function callback.
- */
-typedef size_t (*write_fn_t)( void*, size_t, size_t, void* );
-
-/**
- * Creates a new, initialized cURL instance.
- *
- * @throws exception upon failure.
- */
-CURL* create( char const *uri, write_fn_t fn, void *data );
-
-/**
- * Destroys a cURL instance.
- *
- * @param instance A cURL instance. If \c NULL, does nothing.
- */
-void destroy( CURL *instance );
-
-////////// streambuf //////////////////////////////////////////////////////////
-
-/**
- * A curl::streambuf is-a std::streambuf for streaming the contents of URI
- * using cURL. However, do not use this class directly. Use uri::streambuf
- * instead.
- */
-class streambuf : public std::streambuf {
-public:
- /**
- * Constructs a %streambuf.
- */
- streambuf();
-
- /**
- * Constructs a %streambuf and opens a connection to the server hosting the
- * given URI for subsequent streaming.
- *
- * @param uri The URI to stream.
- */
- streambuf( char const *uri );
-
- /**
- * Constructs a %streambuf using an existing CURL object.
- *
- * @param curl The CURL object to use. This %streambuf takes ownership of
- * it.
- */
- streambuf( CURL *curl );
-
- /**
- * Destroys a %streambuf.
- */
- ~streambuf();
-
- /**
- * Opens a connection to the server hosting the given URI for subsequent
- * streaming.
- *
- * @param uri The URI to stream.
- * @throws exception upon failure.
- */
- void open( char const *uri );
-
- /**
- * Tests whether the buffer is open.
- *
- * @return Returns \c true only if the buffer is open.
- */
- bool is_open() const {
- return !!curl_;
- }
-
- /**
- * Closes this %streambuf.
- */
- void close();
-
- /**
- * Gets the CURL object in use.
- *
- * @return Return said CURL object.
- */
- CURL* curl() const {
- return curl_;
- }
-
- /**
- * Provide a InformDataRead that will get callbacks about read events.
- */
- void setInformer( http_client::InformDataRead *aInformer ) {
- theInformer = aInformer;
- }
-
- /**
- * Specify whether this streambuf has memory ownership over the
- * InformDataRead it has been passed. You can use this if, for example,
- * the lifetime of the streambuf will extend past the lifetime of the
- * object which created the InformDataRead.
- */
- void setOwnInformer( bool aOwnInformer ) {
- theOwnInformer = aOwnInformer;
- }
-
- int multi_perform();
-
-protected:
- // inherited
- std::streamsize showmanyc();
- int_type underflow();
-
-private:
- void curl_read();
- static size_t curl_write_callback( void*, size_t, size_t, void* );
-
- void init();
- void init_curlm();
-
- char *buf_;
- std::streamsize buf_capacity_;
- std::streamoff buf_len_;
-
- CURL *curl_;
- CURLM *curlm_;
- int curl_running_;
- http_client::InformDataRead *theInformer;
- bool theOwnInformer;
-
- // forbid
- streambuf( streambuf const& );
- streambuf& operator=( streambuf const& );
-#ifdef WIN32
- SOCKET dummy_socket_;
-#endif /* WIN32 */
-};
-
-///////////////////////////////////////////////////////////////////////////////
-
-} // namespace curl
-} // namespace zorba
-#endif /* ZORBA_CURL_UTIL_H */
-/* vim:set et sw=2 ts=2: */
=== modified file 'modules/http-client/json/http-client.xq.src/http_response_parser.cpp'
--- modules/http-client/json/http-client.xq.src/http_response_parser.cpp 2013-09-23 09:11:02 +0000
+++ modules/http-client/json/http-client.xq.src/http_response_parser.cpp 2014-01-09 19:12:12 +0000
@@ -36,9 +36,10 @@
#include <zorba/xquery_functions.h>
#include <zorba/internal/unique_ptr.h>
+#include "curl_streambuf.h"
+
#include "http_response_parser.h"
#include "http_request_handler.h"
-#include "curl_stream_buffer.h"
namespace zorba {
@@ -94,10 +95,10 @@
if (!t.empty())
{
if (t[0] == '"' && t[t.length()-1] == '"')
- {
+ {
t.erase( 0, 1 );
- t.erase(t.length() -1, 1);
- }
+ t.erase(t.length() -1, 1);
+ }
*charset = t;
}
}
@@ -129,11 +130,11 @@
int HttpResponseParser::parse()
{
- theStreamBuffer->setInformer(this);
+ theStreamBuffer->set_listener(this);
theHandler.begin();
bool lStatusAndMesssageParsed = false;
int lCode = 0;
- lCode = theStreamBuffer->multi_perform();
+ lCode = theStreamBuffer->curl_multi_info_read(false);
if (lCode)
return lCode;
if (!theStatusOnly) {
@@ -203,7 +204,7 @@
return lCode;
}
- void HttpResponseParser::beforeRead()
+ void HttpResponseParser::curl_read(void*,size_t)
{
if (theInsideRead) {
return;
@@ -218,10 +219,6 @@
theHandler.beginBody(theCurrentContentType, "", NULL);
}
- void HttpResponseParser::afterRead()
- {
- }
-
void HttpResponseParser::registerHandler()
{
curl_easy_setopt(theCurl, CURLOPT_HEADERFUNCTION, &curl_headerfunction);
@@ -343,7 +340,7 @@
// theStreamBuffer. Therefore, this HttpResponseParser object is no longer
// "self-contained". We delegate ownership of ourself to theStreamBuffer
// and mark ourselves as no longer being self-contained.
- theStreamBuffer->setOwnInformer(true);
+ theStreamBuffer->set_listener(this, true);
theSelfContained = false;
// The ownership of theStreamBuffer, in turn, is delegated to the
=== modified file 'modules/http-client/json/http-client.xq.src/http_response_parser.h'
--- modules/http-client/json/http-client.xq.src/http_response_parser.h 2013-07-19 17:36:52 +0000
+++ modules/http-client/json/http-client.xq.src/http_response_parser.h 2014-01-09 19:12:12 +0000
@@ -19,26 +19,21 @@
#include <string>
#include <map>
-#include <curl/curl.h>
+#include "curl_streambuf.h"
-#include "inform_data_read.h"
#include "error_thrower.h"
#include "http_response_handler.h"
namespace zorba {
class Item;
-namespace curl {
- class streambuf;
-}
-
namespace http_client
{
void parse_content_type( std::string const &s, std::string *mime_type, std::string *charset );
class HttpResponseHandler;
- class HttpResponseParser : public InformDataRead {
+ class HttpResponseParser : public curl::listener {
private:
HttpResponseHandler& theHandler;
CURL* theCurl;
@@ -49,7 +44,7 @@
headers_type theHeaders;
int theStatus;
std::string theMessage;
- zorba::curl::streambuf* theStreamBuffer;
+ curl::streambuf* theStreamBuffer;
std::string theId;
std::string theDescription;
bool theInsideRead;
@@ -76,8 +71,7 @@
* will return false.
*/
bool selfContained() { return theSelfContained; }
- virtual void beforeRead();
- virtual void afterRead();
+ virtual void curl_read(void*,size_t);
private:
void registerHandler();
void parseStatusAndMessage(std::string const &aHeader);
=== removed file 'modules/http-client/json/http-client.xq.src/inform_data_read.cpp'
--- modules/http-client/json/http-client.xq.src/inform_data_read.cpp 2013-06-14 16:32:01 +0000
+++ modules/http-client/json/http-client.xq.src/inform_data_read.cpp 1970-01-01 00:00:00 +0000
@@ -1,22 +0,0 @@
-/*
- * Copyright 2006-2013 The FLWOR Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "inform_data_read.h"
-
-namespace zorba { namespace http_client {
- InformDataRead::~InformDataRead()
- {
- }
-}} //namespace zorba, http_client
=== removed file 'modules/http-client/json/http-client.xq.src/inform_data_read.h'
--- modules/http-client/json/http-client.xq.src/inform_data_read.h 2013-06-14 16:32:01 +0000
+++ modules/http-client/json/http-client.xq.src/inform_data_read.h 1970-01-01 00:00:00 +0000
@@ -1,27 +0,0 @@
-/*
- * Copyright 2006-2013 The FLWOR Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef INFORM_DATA_READ_H
-#define INFORM_DATA_READ_H
-namespace zorba { namespace http_client {
- class InformDataRead {
- public:
- virtual ~InformDataRead();
- public:
- virtual void beforeRead() = 0;
- virtual void afterRead() = 0;
- };
-}} //namespace zorba, http_client
-#endif //INFORM_DATA_READ_H
=== added directory 'modules/util-curl'
=== added file 'modules/util-curl/CMakeLists.txt'
--- modules/util-curl/CMakeLists.txt 1970-01-01 00:00:00 +0000
+++ modules/util-curl/CMakeLists.txt 2014-01-09 19:12:12 +0000
@@ -0,0 +1,50 @@
+# Copyright 2006-2012 The FLWOR Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PROJECT(ZorbaUtilCurlModule)
+
+IF (ZORBA_SUPPRESS_CURL)
+ MESSAGE(STATUS "ZORBA_SUPPRESS_CURL is true - not searching for cURL library")
+ELSE (ZORBA_SUPPRESS_CURL)
+ MESSAGE(STATUS "Looking for cURL")
+ FIND_PACKAGE(CURL)
+
+ IF (CURL_FOUND)
+ MESSAGE(STATUS "Found cURL library -- " ${CURL_LIBRARIES})
+ INCLUDE_DIRECTORIES(BEFORE SYSTEM "${CURL_INCLUDE_DIR}")
+ SET(requiredlibs ${requiredlibs} "${CURL_LIBRARIES}")
+
+ SET(ZORBA_PROJECT_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/include")
+ INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/include")
+ INSTALL(FILES include/curl_streambuf.h DESTINATION include)
+
+ ADD_SUBDIRECTORY("src")
+
+ IF (WIN32) # Copy certificates for windows only
+ IF (MSVC_IDE)
+ SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/${CMAKE_BUILD_TYPE}/cacert.pem")
+ ELSE (MSVC_IDE)
+ SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/cacert.pem")
+ ENDIF (MSVC_IDE)
+ CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/cacert.pem" ${CACERT_DESTINATION} COPYONLY)
+ INSTALL(FILES ${CACERT_DESTINATION} DESTINATION bin)
+ ENDIF (WIN32)
+
+ ELSE (CURL_FOUND)
+ MESSAGE(STATUS "The cURL library was not found")
+ ENDIF (CURL_FOUND)
+ENDIF (ZORBA_SUPPRESS_CURL)
+SET(ZORBA_HAVE_CURL ${CURL_FOUND} CACHE BOOL "Whether Zorba found cURL" FORCE)
+
+# vim:set et sw=2 ts=2:
=== added file 'modules/util-curl/ZorbaUtilCurlModuleConfig.cmake'
--- modules/util-curl/ZorbaUtilCurlModuleConfig.cmake 1970-01-01 00:00:00 +0000
+++ modules/util-curl/ZorbaUtilCurlModuleConfig.cmake 2014-01-09 19:12:12 +0000
@@ -0,0 +1,19 @@
+# Copyright 2006-2013 The FLWOR Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SET(ZorbaUtilCurlModule_FOUND TRUE)
+SET(ZorbaUtilCurlModule_INCLUDES "../util-curl/include")
+SET(ZorbaUtilCurlModule_INCLUDE_DIRS "${ZorbaUtilCurlModule_INCLUDES}")
+SET(ZorbaUtilCurlModule_LIBS util-curl)
+SET(ZorbaUtilCurlModule_LIBRARIES "${ZorbaUtilCurlModule_LIBS}")
=== added directory 'modules/util-curl/include'
=== added file 'modules/util-curl/include/curl_streambuf.h'
--- modules/util-curl/include/curl_streambuf.h 1970-01-01 00:00:00 +0000
+++ modules/util-curl/include/curl_streambuf.h 2014-01-09 19:12:12 +0000
@@ -0,0 +1,265 @@
+/*
+ * Copyright 2006-2013 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+#ifndef ZORBA_CURL_STREAMBUF_API_H
+#define ZORBA_CURL_STREAMBUF_API_H
+
+// standard
+#include <cstdlib>
+#include <exception>
+#include <streambuf>
+#include <string>
+#ifdef WIN32
+#include <windows.h>
+#endif /* WIN32 */
+
+// libcurl
+#include <curl/curl.h>
+
+// Zorba
+#include <zorba/util/fs_util.h>
+
+namespace zorba {
+namespace curl {
+
+///////////////////////////////////////////////////////////////////////////////
+
+/**
+ * If defined, all calls to cURL wrapped by either ZORBA_CURL_ASSERT() or
+ * ZORBA_CURLM_ASSERT() will be printed to standard error.
+ */
+//#define ZORBA_TRACE_LIBCURL_CALLS 1
+
+#ifdef ZORBA_TRACE_LIBCURL_CALLS
+# define ZORBA_CURL_ECHO(CURL_FN) \
+ std::cerr << zorba::fs::base_name( __FILE__ ) << ':' << __LINE__ << ": " << #CURL_FN << std::endl
+#else
+# define ZORBA_CURL_ECHO(CURL_FN) (void)0
+#endif /* ZORBA_TRACE_LIBCURL_CALLS */
+
+#define ZORBA_CURL_ASSERT(EXPR) \
+ do { \
+ ZORBA_CURL_ECHO( EXPR ); \
+ if ( CURLcode const code##__LINE__ = (EXPR) ) \
+ throw zorba::curl::exception( #EXPR, "", code##__LINE__ ); \
+ } while (0)
+
+#define ZORBA_CURLM_ASSERT(EXPR) \
+ do { \
+ ZORBA_CURL_ECHO( EXPR ); \
+ if ( CURLMcode const code##__LINE__ = (EXPR) ) \
+ if ( code##__LINE__ != CURLM_CALL_MULTI_PERFORM ) \
+ throw zorba::curl::exception( #EXPR, "", code##__LINE__ ); \
+ } while (0)
+
+////////// exception //////////////////////////////////////////////////////////
+
+/**
+ * A curl::exception is-an exception for cURL errors. These are thrown instead
+ * of simply returning error codes (that are often ignored).
+ */
+class exception : public std::exception {
+public:
+ exception( char const *function, char const *uri, char const *msg = 0 );
+ exception( char const *function, char const *uri, CURLcode code );
+ exception( char const *function, char const *uri, CURLMcode code );
+ ~exception() throw();
+
+ /**
+ * Gets the cURL error code (returned from the "easy" interface).
+ *
+ * @return Returns said error code or 0 if none.
+ */
+ CURLcode curl_code() const {
+ return curl_code_;
+ }
+
+ /**
+ * Gets the cURL error code (returned from the "multi" interface).
+ *
+ * @return Returns said error code or 0 if none.
+ */
+ CURLMcode curlm_code() const {
+ return curlm_code_;
+ }
+
+ // inherited
+ virtual const char* what() const throw();
+
+private:
+ CURLcode curl_code_;
+ CURLMcode curlm_code_;
+ std::string msg_;
+};
+
+////////// streambuf //////////////////////////////////////////////////////////
+
+/**
+ * A listener can be used to "listen" to the raw data that cURL reads at the
+ * time it reads it.
+ */
+struct listener {
+ virtual ~listener();
+
+ /**
+ * This is called whenever cURL reads data and gives it to the streambuf.
+ *
+ * @param data A pointer to the data read.
+ * @param size The number of bytes read.
+ */
+ virtual void curl_read( void *data, size_t size ) = 0;
+};
+
+////////// streambuf //////////////////////////////////////////////////////////
+
+/**
+ * A curl::streambuf is-a std::streambuf for streaming the contents of URI
+ * using cURL. Note that this streambuf can be used only for reading
+ * (downloading) and not writing (uploading) via, say, FTP or HTTP.
+ */
+class streambuf : public std::streambuf {
+public:
+ /**
+ * Constructs a %streambuf.
+ */
+ streambuf();
+
+ /**
+ * Constructs a %streambuf and opens a connection to the server hosting the
+ * given URI for subsequent streaming.
+ *
+ * @param uri The URI to stream.
+ */
+ streambuf( char const *uri );
+
+ /**
+ * Constructs a %streambuf using an existing CURL object.
+ *
+ * @param curl The CURL object to use. This %streambuf takes ownership of
+ * it.
+ */
+ streambuf( CURL *curl );
+
+ /**
+ * Destroys a %streambuf.
+ */
+ ~streambuf();
+
+ /**
+ * Opens a connection to the server hosting the given URI for subsequent
+ * streaming.
+ *
+ * @param uri The URI to stream.
+ * @throws exception upon failure.
+ */
+ void open( char const *uri );
+
+ /**
+ * Tests whether the buffer is open.
+ *
+ * @return Returns \c true only if the buffer is open.
+ */
+ bool is_open() const {
+ return !!curl_;
+ }
+
+ /**
+ * Closes this %streambuf.
+ */
+ void close();
+
+ /**
+ * Gets the CURL ("easy") object in use.
+ *
+ * @return Return said CURL object.
+ */
+ CURL* curl() const {
+ return curl_;
+ }
+
+ /**
+ * Gets the CURLM ("multi") object in use.
+ *
+ * @return Return said CURLM object.
+ */
+ CURLM* curlm() const {
+ return curlm_;
+ }
+
+ /**
+ * Resets all options on the CURL object in use.
+ */
+ void curl_reset() {
+ curl_init();
+ }
+
+ /**
+ * Sets/clears CURL verbose mode.
+ *
+ * @param verbose If \c true, sets verbose mode; otherwise clears it.
+ */
+ void curl_verbose( bool verbose );
+
+public:
+ CURLcode curl_multi_info_read( bool throw_on_error = true );
+
+ void set_listener( listener *l, bool take_ownership = false );
+
+protected:
+ // inherited
+ std::streamsize showmanyc();
+ int_type underflow();
+ std::streamsize xsgetn( char_type*, std::streamsize );
+
+private:
+ void curl_create();
+ void curl_destroy();
+ void curl_init();
+ void curlm_init();
+ void curl_io( size_t* );
+ void curl_write();
+ static size_t curl_write_callback( void*, size_t, size_t, void* );
+ void init();
+
+ struct gbuf {
+ char *ptr_;
+ size_t capacity_, len_;
+ gbuf() : ptr_( 0 ), capacity_( 0 ), len_( 0 ) { }
+ ~gbuf() { free( ptr_ ); }
+ };
+
+ CURL *curl_;
+ CURLM *curlm_;
+ int curl_running_;
+ gbuf gbuf_;
+ listener *listener_;
+ bool listener_owner_;
+ bool verbose_;
+
+ // forbid
+ streambuf( streambuf const& );
+ streambuf& operator=( streambuf const& );
+#ifdef WIN32
+ SOCKET dummy_socket_;
+#endif /* WIN32 */
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace curl
+} // namespace zorba
+#endif /* ZORBA_CURL_STREAMBUF_API_H */
+/* vim:set et sw=2 ts=2: */
=== added directory 'modules/util-curl/src'
=== added file 'modules/util-curl/src/CMakeLists.txt'
--- modules/util-curl/src/CMakeLists.txt 1970-01-01 00:00:00 +0000
+++ modules/util-curl/src/CMakeLists.txt 2014-01-09 19:12:12 +0000
@@ -0,0 +1,36 @@
+# Copyright 2006-2010 The FLWOR Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DECLARE_ZORBA_MODULE(FILE "util-curl.xq" VERSION 1.0
+ URI "http://zorba.io/modules/util-curl";)
+
+# The important stuff is the library that we install in Zorba's default lib
+# directory.
+ADD_LIBRARY(util-curl SHARED curl_streambuf.cpp util-curl.cpp)
+TARGET_LINK_LIBRARIES(util-curl zorba_${ZORBA_STORE_NAME} ${CURL_LIBRARIES})
+INSTALL(TARGETS util-curl
+ RUNTIME DESTINATION bin
+ LIBRARY DESTINATION lib${LIB_SUFFIX}
+ ARCHIVE DESTINATION lib)
+
+# MAC OS X only property
+# This is required to make sure that the library has the correct install name
+# after installation such that dependent modules always find it.
+SET_TARGET_PROPERTIES(util-curl PROPERTIES INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib")
+
+# Set this in the parent scope so it will be put into our Config.cmake file.
+# As with many other things, this doesn't work when the module is installed.
+SET(ZORBA_PROJECT_LIBRARIES util-curl PARENT_SCOPE)
+
+# vim:set et sw=2 ts=2:
=== added file 'modules/util-curl/src/curl_streambuf.cpp'
--- modules/util-curl/src/curl_streambuf.cpp 1970-01-01 00:00:00 +0000
+++ modules/util-curl/src/curl_streambuf.cpp 2014-01-09 19:12:12 +0000
@@ -0,0 +1,386 @@
+/*
+ * Copyright 2006-2013 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// standard
+#include <algorithm>
+#include <cstdlib>
+#include <cstring> /* for memcpy(3) */
+#include <iostream>
+#ifndef WIN32
+#include <cerrno>
+#include <sys/time.h>
+#endif /* WIN32 */
+
+// libcurl
+#include <curl/multi.h>
+
+// Zorba
+#include "curl_streambuf.h"
+
+using namespace std;
+
+namespace zorba {
+namespace curl {
+
+///////////////////////////////////////////////////////////////////////////////
+
+exception::exception( char const *function, char const *uri, char const *msg ) :
+ std::exception(),
+ curl_code_( CURLE_OK ), curlm_code_( CURLM_OK ), msg_( msg )
+{
+}
+
+exception::exception( char const *function, char const *uri, CURLcode code ) :
+ std::exception(),
+ curl_code_( code ), curlm_code_( CURLM_OK ),
+ msg_( curl_easy_strerror( code ) )
+{
+ ostringstream oss;
+ oss << " (CURLcode " << (int)code << ')';
+ msg_ += oss.str();
+}
+
+exception::exception( char const *function, char const *uri, CURLMcode code ) :
+ std::exception(),
+ curl_code_( CURLE_OK ), curlm_code_( code ),
+ msg_( curl_multi_strerror( code ) )
+{
+ ostringstream oss;
+ oss << " (CURLMcode " << (int)code << ')';
+ msg_ += oss.str();
+}
+
+exception::~exception() throw() {
+ // out-of-line since it's virtual
+}
+
+const char* exception::what() const throw() {
+ return msg_.c_str();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+listener::~listener() {
+ // out-of-line since it's virtual
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+streambuf::streambuf() {
+ init();
+}
+
+streambuf::streambuf( char const *uri ) {
+ init();
+ open( uri );
+}
+
+streambuf::streambuf( CURL *curl ) {
+ init();
+ curl_ = curl;
+ curl_init();
+ curlm_init();
+}
+
+streambuf::~streambuf() {
+ close();
+#ifdef WIN32
+ closesocket( dummy_socket_ );
+#endif /* WIN32 */
+ if ( listener_owner_ )
+ delete listener_;
+}
+
+void streambuf::close() {
+ if ( curl_ ) {
+ if ( curlm_ ) {
+ curl_multi_remove_handle( curlm_, curl_ );
+ curl_multi_cleanup( curlm_ );
+ curlm_ = 0;
+ }
+ curl_destroy();
+ }
+}
+
+void streambuf::curl_create() {
+ //
+ // Having cURL initialization wrapped by a class and using a singleton static
+ // instance guarantees that cURL is initialized exactly once before use and
+ // and also is cleaned-up at program termination (when destructors for static
+ // objects are called).
+ //
+ struct curl_initializer {
+ curl_initializer() {
+ ZORBA_CURL_ASSERT( curl_global_init( CURL_GLOBAL_ALL ) );
+ }
+ ~curl_initializer() {
+ curl_global_cleanup();
+ }
+ };
+ static curl_initializer initializer;
+
+ curl_ = curl_easy_init();
+ if ( !curl_ )
+ throw exception( "curl_easy_init()", "", "" );
+ try {
+ curl_init();
+ }
+ catch ( ... ) {
+ curl_destroy();
+ throw;
+ }
+}
+
+void streambuf::curl_destroy() {
+ if ( curl_ ) {
+ curl_easy_reset( curl_ );
+ curl_easy_cleanup( curl_ );
+ curl_ = 0;
+ }
+}
+
+void streambuf::curl_verbose( bool verbose ) {
+ verbose_ = verbose;
+ if ( curl_ )
+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_VERBOSE, verbose_ ? 1 : 0 ) );
+}
+
+void streambuf::curl_init() {
+ curl_easy_reset( curl_ );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_FOLLOWLOCATION, 1 ) );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_MAXREDIRS, 50L ) );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_NOPROGRESS, 1L ) );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_TCP_KEEPALIVE, 1L ) );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_WRITEDATA, this ) );
+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_WRITEFUNCTION, curl_write_callback ) );
+
+ if ( verbose_ )
+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_VERBOSE, 1 ) );
+}
+
+void streambuf::curl_io( size_t *len_ptr ) {
+ *len_ptr = 0;
+ while ( curl_running_ && !*len_ptr ) {
+ fd_set fd_read, fd_write, fd_except;
+ FD_ZERO( &fd_read );
+ FD_ZERO( &fd_write );
+ FD_ZERO( &fd_except );
+ int max_fd = -1;
+#ifdef WIN32
+ //
+ // Windows does not like a call to select where all arguments are 0, so we
+ // just add a dummy socket to make the call to select happy.
+ //
+ FD_SET( dummy_socket_, &fd_read );
+#endif /* WIN32 */
+ ZORBA_CURLM_ASSERT(
+ curl_multi_fdset( curlm_, &fd_read, &fd_write, &fd_except, &max_fd )
+ );
+
+ //
+ // Note that the fopen.c sample code is unnecessary at best or wrong at
+ // worst; see: http://curl.haxx.se/mail/lib-2011-05/0011.html
+ //
+ timeval timeout;
+ long curl_timeout_ms;
+ ZORBA_CURLM_ASSERT( curl_multi_timeout( curlm_, &curl_timeout_ms ) );
+ if ( curl_timeout_ms > 0 ) {
+ timeout.tv_sec = curl_timeout_ms / 1000;
+ timeout.tv_usec = curl_timeout_ms % 1000 * 1000;
+ } else {
+ //
+ // From curl_multi_timeout(3):
+ //
+ // Note: if libcurl returns a -1 timeout here, it just means that
+ // libcurl currently has no stored timeout value. You must not wait
+ // too long (more than a few seconds perhaps) before you call
+ // curl_multi_perform() again.
+ //
+ // So we just pick some not-too-long default.
+ //
+ timeout.tv_sec = 1;
+ timeout.tv_usec = 0;
+ }
+
+ switch ( select( max_fd + 1, &fd_read, &fd_write, &fd_except, &timeout ) ) {
+ case -1: // select error
+#ifdef WIN32
+ char err_buf[8];
+ sprintf( err_buf, "%d", WSAGetLastError() );
+ throw exception( "select()", "", err_buf );
+#else
+ throw exception( "select()", "", strerror( errno ) );
+#endif /* WIN32 */
+ case 0: // timeout
+ // no break;
+ default:
+ CURLMcode code;
+ do {
+ code = curl_multi_perform( curlm_, &curl_running_ );
+ } while ( code == CURLM_CALL_MULTI_PERFORM );
+ ZORBA_CURLM_ASSERT( code );
+ } // switch
+ } // while
+}
+
+CURLcode streambuf::curl_multi_info_read( bool throw_on_error ) {
+ underflow();
+ CURLMsg *msg;
+ int msgs_in_q;
+ CURLcode code = CURLE_OK;
+ while ( (msg = ::curl_multi_info_read( curlm_, &msgs_in_q )) )
+ if ( msg->msg == CURLMSG_DONE )
+ code = msg->data.result;
+ if ( code && throw_on_error )
+ throw exception( "curl_multi_info_read()", "", code );
+ return code;
+}
+
+size_t streambuf::curl_write_callback( void *ptr, size_t size, size_t nmemb,
+ void *data ) {
+ size *= nmemb;
+ streambuf *const that = static_cast<streambuf*>( data );
+
+ if ( that->listener_ )
+ that->listener_->curl_read( ptr, size );
+
+ size_t const gbuf_free = that->gbuf_.capacity_ - that->gbuf_.len_;
+ if ( size > gbuf_free ) {
+ size_t new_capacity = that->gbuf_.capacity_ + size - gbuf_free;
+ if ( void *const new_buf = realloc( that->gbuf_.ptr_, new_capacity ) ) {
+ that->gbuf_.ptr_ = static_cast<char*>( new_buf );
+ that->gbuf_.capacity_ = new_capacity;
+ } else
+ throw exception( "realloc()", "" );
+ }
+ ::memcpy( that->gbuf_.ptr_ + that->gbuf_.len_, ptr, size );
+ that->gbuf_.len_ += size;
+ return size;
+}
+
+void streambuf::init() {
+ curl_ = 0;
+ curlm_ = 0;
+ curl_running_ = 0;
+#ifdef WIN32
+ dummy_socket_ = socket( AF_INET, SOCK_DGRAM, 0 );
+ if ( dummy_socket_ == CURL_SOCKET_BAD || dummy_socket_ == INVALID_SOCKET )
+ throw exception( "socket()", "" );
+#endif /* WIN32 */
+ listener_ = 0;
+ listener_owner_ = false;
+}
+
+void streambuf::curlm_init() {
+ //
+ // Lie about cURL running initially so the while-loop in curl_io() will run
+ // at least once.
+ //
+ curl_running_ = 1;
+
+ //
+ // Set the "get" pointer to the end (gptr() == egptr()) so a call to
+ // underflow() and initial data read will be triggered.
+ //
+ gbuf_.len_ = gbuf_.capacity_;
+ setg( gbuf_.ptr_, gbuf_.ptr_ + gbuf_.len_, gbuf_.ptr_ + gbuf_.len_ );
+
+ //
+ // Clean-up has to be done here with try/catch (as opposed to relying on the
+ // destructor) because open() can be called from the constructor. If an
+ // exception is thrown, the constructor will not have completed, hence the
+ // object will not have been fully constructed; therefore the destructor will
+ // not be called.
+ //
+ try {
+ if ( !(curlm_ = curl_multi_init()) )
+ throw exception( "curl_multi_init()", "" );
+ try {
+ ZORBA_CURLM_ASSERT( curl_multi_add_handle( curlm_, curl_ ) );
+ }
+ catch ( ... ) {
+ curl_multi_cleanup( curlm_ );
+ curlm_ = 0;
+ throw;
+ }
+ }
+ catch ( ... ) {
+ curl_destroy();
+ throw;
+ }
+}
+
+void streambuf::open( char const *uri ) {
+ if ( !curl_ ) {
+ curl_create();
+ curlm_init();
+ }
+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_URL, uri ) );
+}
+
+void streambuf::set_listener( listener *l, bool take_ownership ) {
+ if ( listener_owner_ )
+ delete listener_;
+ listener_ = l;
+ listener_owner_ = take_ownership;
+}
+
+streamsize streambuf::showmanyc() {
+ return egptr() - gptr();
+}
+
+streambuf::int_type streambuf::underflow() {
+ while ( true ) {
+ if ( gptr() < egptr() )
+ return traits_type::to_int_type( *gptr() );
+ curl_io( &gbuf_.len_ );
+ if ( !gbuf_.len_ )
+ return traits_type::eof();
+ setg( gbuf_.ptr_, gbuf_.ptr_, gbuf_.ptr_ + gbuf_.len_ );
+ }
+}
+
+streamsize streambuf::xsgetn( char_type *to, streamsize size ) {
+ streamsize return_size = 0;
+
+ if ( streamsize const gsize = egptr() - gptr() ) {
+ streamsize const n = min( gsize, size );
+ traits_type::copy( to, gptr(), static_cast<size_t>( n ) );
+ gbump( static_cast<int>( n ) );
+ to += n;
+ size -= n, return_size += n;
+ }
+
+ while ( size > 0 ) {
+ streamsize const get = min( (streamsize)gbuf_.capacity_, size );
+ curl_io( &gbuf_.len_ );
+ if ( !gbuf_.len_ )
+ break;
+ setg( gbuf_.ptr_, gbuf_.ptr_, gbuf_.ptr_ + gbuf_.len_ );
+ streamsize const n = min( (streamsize)gbuf_.len_, size );
+ traits_type::copy( to, gptr(), n );
+ gbump( static_cast<int>( n ) );
+ to += n;
+ size -= n, return_size += n;
+ }
+ return return_size;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace curl
+} // namespace zorba
+/* vim:set et sw=2 ts=2: */
=== added file 'modules/util-curl/src/util-curl.cpp'
--- modules/util-curl/src/util-curl.cpp 1970-01-01 00:00:00 +0000
+++ modules/util-curl/src/util-curl.cpp 2014-01-09 19:12:12 +0000
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2006-2008 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <zorba/external_module.h>
+
+namespace zorba {
+namespace util_curl {
+
+///////////////////////////////////////////////////////////////////////////////
+
+class module : public ExternalModule {
+public:
+ // inherited
+ virtual void destroy();
+ virtual ExternalFunction* getExternalFunction( String const& );
+ virtual String getURI() const;
+};
+
+void module::destroy() {
+ delete this;
+}
+
+ExternalFunction* module::getExternalFunction( String const& ) {
+ return 0;
+}
+
+String module::getURI() const {
+ return "http://zorba.io/modules/util-curl";;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace util_curl
+} // namespace zorba
+
+#ifdef WIN32
+# define DLL_EXPORT __declspec(dllexport)
+#else
+# define DLL_EXPORT __attribute__ ((visibility("default")))
+#endif
+
+extern "C" DLL_EXPORT zorba::ExternalModule* createModule() {
+ return new zorba::util_curl::module();
+}
+
+/* vim:set et sw=2 ts=2: */
=== added file 'modules/util-curl/src/util-curl.xq'
--- modules/util-curl/src/util-curl.xq 1970-01-01 00:00:00 +0000
+++ modules/util-curl/src/util-curl.xq 2014-01-09 19:12:12 +0000
@@ -0,0 +1,34 @@
+xquery version "1.0";
+
+(:
+ : Copyright 2006-2013 The FLWOR Foundation.
+ :
+ : Licensed under the Apache License, Version 2.0 (the "License");
+ : you may not use this file except in compliance with the License.
+ : You may obtain a copy of the License at
+ :
+ : http://www.apache.org/licenses/LICENSE-2.0
+ :
+ : Unless required by applicable law or agreed to in writing, software
+ : distributed under the License is distributed on an "AS IS" BASIS,
+ : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ : See the License for the specific language governing permissions and
+ : limitations under the License.
+ :)
+
+(:~
+ : This module provides common functionality for modules that use libcurl.
+ : <p/>
+ : Modules using libcurl implementations must import this module
+ : to specify the dependency.
+ :
+ : @author Paul J. Lucas
+ : @library <a href="http://curl.haxx.se/download.html";>libcurl</a>
+ : @project Zorba/cURL Utility
+ :)
+module namespace util-curl = "http://zorba.io/modules/util-curl";;
+
+declare namespace ver = "http://zorba.io/options/versioning";;
+declare option ver:module-version "1.0";
+
+(: vim:set et sw=2 ts=2: :)
=== modified file 'src/api/CMakeLists.txt'
--- src/api/CMakeLists.txt 2013-09-16 09:08:27 +0000
+++ src/api/CMakeLists.txt 2014-01-09 19:12:12 +0000
@@ -59,6 +59,7 @@
auditimpl.cpp
store_consts.cpp
streambuf.cpp
+ mem_streambuf.cpp
transcode_streambuf.cpp
uuid.cpp
module_info_impl.cpp
=== modified file 'src/api/base64_util.cpp'
--- src/api/base64_util.cpp 2013-07-30 18:35:31 +0000
+++ src/api/base64_util.cpp 2014-01-09 19:12:12 +0000
@@ -22,8 +22,8 @@
// Zorba
#include <zorba/util/base64_util.h>
+#include <zorba/util/mem_streambuf.h>
#include "util/ascii_util.h"
-#include "util/mem_streambuf.h"
#include "util/string_util.h"
using namespace std;
=== modified file 'src/api/hexbinary_util.cpp'
--- src/api/hexbinary_util.cpp 2013-08-02 14:55:29 +0000
+++ src/api/hexbinary_util.cpp 2014-01-09 19:12:12 +0000
@@ -22,7 +22,7 @@
// Zorba
#include <zorba/util/hexbinary_util.h>
-#include "util/mem_streambuf.h"
+#include <zorba/util/mem_streambuf.h>
#include "util/string_util.h"
using namespace std;
=== modified file 'src/api/item.cpp'
--- src/api/item.cpp 2013-08-01 17:38:35 +0000
+++ src/api/item.cpp 2014-01-09 19:12:12 +0000
@@ -534,7 +534,7 @@
Item
-Item::getObjectValue(String aName) const
+Item::getObjectValue(String const &aName) const
{
ITEM_TRY
SYNC_CODE(AutoLock lock(GENV_STORE.getGlobalLock(), Lock::READ);)
=== renamed file 'src/util/mem_streambuf.cpp' => 'src/api/mem_streambuf.cpp'
--- src/util/mem_streambuf.cpp 2013-08-01 00:31:20 +0000
+++ src/api/mem_streambuf.cpp 2014-01-09 19:12:12 +0000
@@ -18,11 +18,10 @@
#include <cstring> /* for memcpy(3) */
#include <zorba/internal/cxx_util.h>
+#include <zorba/util/mem_streambuf.h>
#include "diagnostics/assert.h"
-#include "mem_streambuf.h"
-
using namespace std;
namespace zorba {
=== modified file 'src/diagnostics/dict.cpp'
--- src/diagnostics/dict.cpp 2013-08-02 18:34:59 +0000
+++ src/diagnostics/dict.cpp 2014-01-09 19:12:12 +0000
@@ -68,9 +68,15 @@
char const* lookup( char const *key ) {
typedef pair<entry const*,entry const*> range_type;
+ //
+ // It's possible an exception was thrown after shutdown has already started
+ // in which case the GlobalEnvironment no longer exists.
+ //
+ GlobalEnvironment const *const genv = GENV_PTR;
+ iso639_1::type lang = genv ? genv->get_host_lang() : locale::get_host_lang();
static entry const *begin, *end;
- if ( !begin && !get_dict( GENV.get_host_lang(), &begin, &end ) )
+ if ( !begin && !get_dict( lang, &begin, &end ) )
SET_DICT( en, begin, end );
entry entry_to_find;
=== modified file 'src/runtime/csv/pregenerated/csv.h'
--- src/runtime/csv/pregenerated/csv.h 2013-08-27 23:56:50 +0000
+++ src/runtime/csv/pregenerated/csv.h 2014-01-09 19:12:12 +0000
@@ -31,9 +31,9 @@
#include "runtime/base/narybase.h"
#include <sstream>
#include <vector>
+#include <zorba/util/mem_streambuf.h>
#include "runtime/csv/csv_util.h"
#include "util/csv_parser.h"
-#include "util/mem_streambuf.h"
#include "zorbatypes/zstring.h"
=== modified file 'src/runtime/json/json_impl.cpp'
--- src/runtime/json/json_impl.cpp 2013-12-12 18:41:52 +0000
+++ src/runtime/json/json_impl.cpp 2014-01-09 19:12:12 +0000
@@ -20,12 +20,12 @@
#include <zorba/diagnostic_list.h>
#include <zorba/internal/cxx_util.h>
+#include <zorba/util/mem_streambuf.h>
#include "runtime/json/json.h"
#include "store/api/item_factory.h"
#include "system/globalenv.h"
#include "util/ascii_util.h"
-#include "util/mem_streambuf.h"
#include "util/stream_util.h"
#include "jsonml_array.h"
=== modified file 'src/runtime/json/snelson.cpp'
--- src/runtime/json/snelson.cpp 2013-08-25 14:32:02 +0000
+++ src/runtime/json/snelson.cpp 2014-01-09 19:12:12 +0000
@@ -21,6 +21,7 @@
#include <zorba/diagnostic_list.h>
#include <zorba/internal/cxx_util.h>
#include <zorba/store_consts.h>
+#include <zorba/util/mem_streambuf.h>
#include "runtime/json/json.h"
#include "store/api/item_factory.h"
@@ -28,7 +29,6 @@
#include "types/root_typemanager.h"
#include "types/typeops.h"
#include "util/json_parser.h"
-#include "util/mem_streambuf.h"
#include "util/stl_util.h"
#include "zorbatypes/decimal.h"
#include "zorbatypes/float.h"
=== modified file 'src/runtime/spec/csv/csv.xml'
--- src/runtime/spec/csv/csv.xml 2013-08-27 23:56:50 +0000
+++ src/runtime/spec/csv/csv.xml 2014-01-09 19:12:12 +0000
@@ -8,9 +8,9 @@
<zorba:header>
<zorba:include form="Angle-bracket">sstream</zorba:include>
<zorba:include form="Angle-bracket">vector</zorba:include>
+ <zorba:include form="Angle-bracket">zorba/util/mem_streambuf.h</zorba:include>
<zorba:include form="Quoted">runtime/csv/csv_util.h</zorba:include>
<zorba:include form="Quoted">util/csv_parser.h</zorba:include>
- <zorba:include form="Quoted">util/mem_streambuf.h</zorba:include>
<zorba:include form="Quoted">zorbatypes/zstring.h</zorba:include>
</zorba:header>
=== modified file 'src/system/globalenv.h'
--- src/system/globalenv.h 2013-09-24 09:48:50 +0000
+++ src/system/globalenv.h 2014-01-09 19:12:12 +0000
@@ -102,6 +102,11 @@
return *m_globalEnv;
}
+ static GlobalEnvironment* getInstancePtr()
+ {
+ return m_globalEnv;
+ }
+
public:
~GlobalEnvironment();
@@ -151,6 +156,7 @@
#define GENV GlobalEnvironment::getInstance()
+#define GENV_PTR GlobalEnvironment::getInstancePtr()
#define GENV_TYPESYSTEM GlobalEnvironment::getInstance().getRootTypeManager()
=== modified file 'src/unit_tests/test_base64_streambuf.cpp'
--- src/unit_tests/test_base64_streambuf.cpp 2013-06-11 23:38:49 +0000
+++ src/unit_tests/test_base64_streambuf.cpp 2014-01-09 19:12:12 +0000
@@ -100,6 +100,13 @@
return b64_str == expected_b64_str;
}
+static void test_instantiate() {
+ base64::auto_attach<ostream> aao;
+ (void)aao;
+ base64::stream<ostream> b64os;
+ (void)b64os;
+}
+
static bool test_put( test const *t ) {
ostringstream oss;
{ // local scope
@@ -128,6 +135,8 @@
namespace UnitTests {
int test_base64_streambuf( int, char*[] ) {
+ test_instantiate();
+
int test_no = 0;
for ( test const *t = tests; t->raw_str; ++t, ++test_no ) {
ASSERT_TRUE_AND_NO_EXCEPTION( test_no, test_getline( t ) );
=== modified file 'src/util/CMakeLists.txt'
--- src/util/CMakeLists.txt 2013-08-21 23:52:57 +0000
+++ src/util/CMakeLists.txt 2014-01-09 19:12:12 +0000
@@ -22,7 +22,6 @@
json_parser.cpp
json_util.cpp
locale.cpp
- mem_streambuf.cpp
stream_util.cpp
string_util.cpp
time_util.cpp
Follow ups
-
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: noreply, 2014-01-13
-
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Zorba Build Bot, 2014-01-13
-
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Zorba Build Bot, 2014-01-13
-
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Zorba Build Bot, 2014-01-11
-
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Zorba Build Bot, 2014-01-11
-
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Zorba Build Bot, 2014-01-11
-
Re: [Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Matthias Brantner, 2014-01-11
-
Re: [Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Matthias Brantner, 2014-01-11
-
Re: [Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Matthias Brantner, 2014-01-11
-
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Paul J. Lucas, 2014-01-10
-
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Paul J. Lucas, 2014-01-10
-
[Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Paul J. Lucas, 2014-01-10
-
Re: [Merge] lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba
From: Paul J. Lucas, 2014-01-10
