linuxdcpp-team team mailing list archive

[noreply@xxxxxxxxxxxxx]

------------------------------------------------------------
revno: 2776
committer: Jacek Sieka <arnetheduck@xxxxxxxxx>
branch nick: dcplusplus
timestamp: Sat 2011-12-31 12:50:24 +0100
message:
  File reader & test utility, wip
added:
  dcpp/FileReader.cpp
  dcpp/FileReader.h
  utils/
  utils/SConscript
  utils/xsum.cpp
modified:
  SConstruct
  dcpp/Util.cpp
  test/SConscript


--
lp:dcplusplus
https://code.launchpad.net/~dcplusplus-team/dcplusplus/trunk

Your team Dcplusplus-team is subscribed to branch lp:dcplusplus.
To unsubscribe from this branch go to https://code.launchpad.net/~dcplusplus-team/dcplusplus/trunk/+edit-subscription

=== modified file 'SConstruct'
--- SConstruct	2011-12-27 19:47:56 +0000
+++ SConstruct	2011-12-31 11:50:24 +0000
@@ -84,6 +84,7 @@
 	BoolVariable('help', 'Build help files (requires i18n=1)', 'yes'),
 	BoolVariable('webhelp', 'Build help files for the web (requires help=1)', 'no'),
 	BoolVariable('test', 'Build test suite', 'no'),
+	BoolVariable('utils', 'Build utils suite', 'no'),
 	('prefix', 'Prefix to use when cross compiling', ''),
 	EnumVariable('arch', 'Target architecture', 'x86', ['x86', 'x64', 'ia64']),
 	BoolVariable('msvcproj', 'Build MSVC project files', 'no'),
@@ -264,8 +265,12 @@
 if dev.env['test']:
 	dev.test = dev.build('test/')
 	env.Default(dev.test)
+elif dev.env['utils']:
+	dev.utils = dev.build('utils/')
+	env.Default(dev.utils)
 else:
 	dev.win32 = dev.build('win32/')
+	
 dev.installer = dev.build('installer/')
 
 dev.finalize()

=== added file 'dcpp/FileReader.cpp'
--- dcpp/FileReader.cpp	1970-01-01 00:00:00 +0000
+++ dcpp/FileReader.cpp	2011-12-31 11:50:24 +0000
@@ -0,0 +1,174 @@
+#include "stdinc.h"
+
+#include "FileReader.h"
+
+#include "File.h"
+#include "Text.h"
+#include "Util.h"
+
+namespace dcpp {
+
+using std::make_pair;
+using std::swap;
+
+namespace {
+static const size_t READ_FAILED = static_cast<size_t>(-1);
+}
+
+size_t FileReader::read(const string& file, const DataCallback& callback) {
+	size_t ret = READ_FAILED;
+
+	if(direct) {
+		ret = readDirect(file, callback);
+	}
+
+	if(ret == READ_FAILED) {
+		ret = readMapped(file, callback);
+
+		if(ret == READ_FAILED) {
+			ret = readCached(file, callback);
+		}
+	}
+
+	return ret;
+}
+
+
+/** Read entire file, never returns READ_FAILED */
+size_t FileReader::readCached(const string& file, const DataCallback& callback) {
+	auto buf = getBuffer(512);
+
+	File f(file, File::READ, File::OPEN | File::SHARED);
+
+	size_t total = 0;
+	size_t n = buf.second;
+	while(f.read(buf.first, n) > 0) {
+		callback(buf.first, n);
+		total += n;
+		n = buf.second;
+	}
+
+	return total;
+}
+
+pair<void*, size_t> FileReader::getBuffer(size_t alignment) {
+	auto block = (((blockSize == 0 ? DEFAULT_BLOCK_SIZE : blockSize) + alignment - 1) / alignment) * alignment;
+
+	buffer.resize(block * 2 + alignment); // Prepare for worst case alignment
+
+	auto start = reinterpret_cast<void*>(((reinterpret_cast<size_t>(&buffer[0]) + alignment - 1) / alignment) * alignment);
+	return make_pair(start, block);
+}
+
+#ifdef _WIN32
+
+struct Handle : boost::noncopyable {
+	Handle(HANDLE h) : h(h) { }
+	~Handle() { ::CloseHandle(h); }
+
+	operator HANDLE() { return h; }
+
+	HANDLE h;
+};
+
+size_t FileReader::readDirect(const string& file, const DataCallback& callback) {
+	DWORD sector = 0, y;
+
+	auto tfile = Text::toT(file);
+
+	if (!::GetDiskFreeSpace(Util::getFilePath(tfile).c_str(), &y, &sector, &y, &y)) {
+		dcdebug("Failed to get sector size: %s\n", Util::translateError(::GetLastError()).c_str());
+		return READ_FAILED;
+	}
+
+	auto tmp = ::CreateFile(tfile.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING,
+		FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, NULL);
+
+	if (tmp == INVALID_HANDLE_VALUE) {
+		dcdebug("Failed to open unbuffered file: %s\n", Util::translateError(::GetLastError()).c_str());
+		return false;
+	}
+
+	Handle h(tmp);
+
+	auto buf = getBuffer(sector);
+	DWORD bufSize = static_cast<DWORD>(buf.second);
+
+	DWORD hn = 0;
+	DWORD rn = 0;
+	uint8_t* hbuf = static_cast<uint8_t*>(buf.first) + bufSize;
+	uint8_t* rbuf = static_cast<uint8_t*>(buf.first);
+	OVERLAPPED over = { 0 };
+
+	// Read the first block
+	auto res = ::ReadFile(h, hbuf, buf.second, &hn, &over);
+
+	if(!res) {
+		auto err = ::GetLastError();
+		if (err == ERROR_HANDLE_EOF) {
+			hn = 0;
+		} else if(err == ERROR_IO_PENDING) {
+			// Finish the read and see how it went
+			if(!GetOverlappedResult(h, &over, &hn, TRUE)) {
+				if (::GetLastError() == ERROR_HANDLE_EOF) {
+					hn = 0;
+				} else {
+					dcdebug("First overlapped read failed: %s\n", Util::translateError(::GetLastError()).c_str());
+					return READ_FAILED;
+				}
+			}
+		}
+	}
+
+	over.Offset = hn;
+
+	for (; hn > 0;) {
+		// Last read returned some bytes, start a new overlapped read
+		res = ::ReadFile(h, rbuf, buf.second, &rn, &over);
+
+		callback(hbuf, hn);
+
+		if (!res) {
+			auto err = ::GetLastError();
+			if(err == ERROR_HANDLE_EOF) {
+				rn = 0;
+			} else if(err == ERROR_IO_PENDING) {
+				// Finish the read
+				if (!GetOverlappedResult(h, &over, &rn, TRUE)) {
+					err = ::GetLastError();
+					if(err != ERROR_HANDLE_EOF) {
+						throw FileException(Util::translateError(err));
+					}
+
+					rn = 0;
+				}
+			} else {
+				throw FileException(Util::translateError(::GetLastError()));
+			}
+		}
+
+		*((uint64_t*)&over.Offset) += rn;
+
+		swap(rbuf, hbuf);
+		swap(rn, hn);
+	}
+
+	return *((uint64_t*)&over.Offset);
+}
+
+size_t FileReader::readMapped(const string& file, const DataCallback& callback) {
+	return READ_FAILED;
+}
+
+#else
+
+size_t FileReader::readDirect(const string& file, const DataCallback& callback) {
+	return READ_FAILED;
+}
+
+size_t FileReader::readMapped(const string& file, const DataCallback& callback) {
+	return READ_FAILED;
+}
+
+#endif
+}

=== added file 'dcpp/FileReader.h'
--- dcpp/FileReader.h	1970-01-01 00:00:00 +0000
+++ dcpp/FileReader.h	2011-12-31 11:50:24 +0000
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2001-2011 Jacek Sieka, arnetheduck on gmail point com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef DCPLUSPLUS_DCPP_CONNECTION_MANAGER_H
+#define DCPLUSPLUS_DCPP_CONNECTION_MANAGER_H
+
+#include <functional>
+#include <string>
+#include <vector>
+
+#include <boost/noncopyable.hpp>
+
+namespace dcpp {
+
+using std::function;
+using std::pair;
+using std::string;
+using std::vector;
+
+/** Helper class for reading an entire file */
+
+class FileReader : boost::noncopyable {
+public:
+
+	enum Strategy {
+		DIRECT,
+		MAPPED,
+		CACHED
+	};
+
+	typedef function<void(void*, size_t)> DataCallback;
+
+	/**
+	 * Set up file reader
+	 * @param direct Bypass system caches - good for reading files which are not in the cache and should not be there (for example when hashing)
+	 * @param blockSize Read block size, 0 = use default
+	 */
+	FileReader(bool direct = false, size_t blockSize = 0) : direct(direct), blockSize(blockSize) { }
+
+	/**
+	 * Read file - callback will be called for each read chunk which may or may not be a multiple of the requested block size.
+	 * @param file File name
+	 * @param callback Called for each block - the memory is owned by the reader object and
+	 * @return The number of bytes actually read
+	 * @throw FileException if the read fails
+	 */
+	size_t read(const string& file, const DataCallback& callback);
+private:
+	static const size_t DEFAULT_BLOCK_SIZE = 64*1024;
+	static const size_t DEFAULT_MMAP_SIZE = 64*1024*1024;
+
+	string file;
+	bool direct;
+	size_t blockSize;
+
+	vector<uint8_t> buffer;
+
+	/** Return an aligned buffer which is at least twice the size of ret.second */
+	pair<void*, size_t> getBuffer(size_t alignment);
+
+	size_t readDirect(const string& file, const DataCallback& callback);
+	size_t readMapped(const string& file, const DataCallback& callback);
+	size_t readCached(const string& file, const DataCallback& callback);
+};
+
+}
+
+#endif

=== modified file 'dcpp/Util.cpp'
--- dcpp/Util.cpp	2011-12-22 22:14:45 +0000
+++ dcpp/Util.cpp	2011-12-31 11:50:24 +0000
@@ -501,7 +501,7 @@
 }
 
 string Util::formatBytes(int64_t aBytes) {
-	char buf[128];
+	char buf[128] = { 0 };
 	if(aBytes < 1024) {
 		snprintf(buf, sizeof(buf), _("%d B"), (int)(aBytes&0xffffffff));
 	} else if(aBytes < 1024*1024) {

=== modified file 'test/SConscript'
--- test/SConscript	2011-10-23 13:22:23 +0000
+++ test/SConscript	2011-12-31 11:50:24 +0000
@@ -39,7 +39,7 @@
 	openssl_lib += env['arch'] + '/'
 env.Append(LIBPATH = [openssl_lib])
 
-ret = env.Program(target, [sources, dev.client, dev.zlib, dev.bzip2, dev.intl])
+ret = env.Program(target, [sources, dev.client, dev.dwarf, dev.zlib, dev.boost, dev.bzip2, dev.geoip, dev.miniupnpc, dev.natpmp, dev.intl])
 
 ret = env.Command(dev.get_target(source_path, 'gtest.passed', in_bin=False), ret[0].abspath, runUnitTest)
 

=== added directory 'utils'
=== added file 'utils/SConscript'
--- utils/SConscript	1970-01-01 00:00:00 +0000
+++ utils/SConscript	2011-12-31 11:50:24 +0000
@@ -0,0 +1,38 @@
+# vim: set filetype=py
+ 
+Import('dev source_path')
+
+env, target, sources = dev.prepare_build(source_path, 'xsum', source_glob="xsum.cpp", in_bin=False)
+
+if 'msvc' in env['TOOLS']:
+	if env['mode'] == 'debug':
+		env.Prepend(LIBS = ['ssleay32d', 'libeay32d'])
+	else:
+		env.Prepend(LIBS = ['ssleay32', 'libeay32'])
+else:
+	env.Prepend(LIBS = ['ssl', 'crypto'])
+
+if 'HAVE_HTMLHELP_H' in env['CPPDEFINES']:
+	env.Append(LIBS='htmlhelp')
+
+env.Append(LIBS = ['comctl32', 'ws2_32', 'ole32', 'gdi32', 'comdlg32', 'iphlpapi', 'winmm', 'shlwapi', 'oleaut32', 'uuid'])
+
+env.Append(CPPPATH = ['#/openssl/include', '#/miniupnpc', '#/dwt/include', '#/', '#/bzip2'])
+
+if '-mwindows' in env['CCFLAGS']:
+	env['CCFLAGS'].remove('-mwindows')
+
+if '-mwindows' in env['LINKFLAGS']:
+	env['LINKFLAGS'].remove('-mwindows')
+
+env.Append(CCFLAGS = ['-mconsole'])	
+env.Append(LINKFLAGS = ['-mconsole'])	
+	
+openssl_lib = '#/openssl/lib/'
+if env['arch'] != 'x86':
+	openssl_lib += env['arch'] + '/'
+env.Append(LIBPATH = [openssl_lib])
+
+ret = env.Program(target, [sources, dev.client, dev.dwarf, dev.zlib, dev.boost, dev.bzip2, dev.geoip, dev.miniupnpc, dev.natpmp, dev.intl])
+
+Return('ret')

=== added file 'utils/xsum.cpp'
--- utils/xsum.cpp	1970-01-01 00:00:00 +0000
+++ utils/xsum.cpp	2011-12-31 11:50:24 +0000
@@ -0,0 +1,49 @@
+#include <iostream>
+
+#include <dcpp/stdinc.h>
+#include <dcpp/FileReader.h>
+#include <dcpp/Util.h>
+
+#include <boost/date_time/posix_time/ptime.hpp>
+using namespace boost::posix_time;
+
+using namespace std;
+using namespace dcpp;
+
+int main(int argc, char** argv)
+{
+	if(argc != 2) {
+		cout << "You need to supply a file name" << endl;
+		return 1;
+	}
+
+	char x[_MAX_PATH] = { 0 };
+	char * tmp;
+	if(!(tmp = _fullpath(x, argv[1], _MAX_PATH))) {
+		cout << "Can't get full path" << endl;
+		return 1;
+	}
+	try {
+		auto start = microsec_clock::universal_time();
+		FileReader fr(true, 0);
+
+		size_t total = 0;
+		fr.read(tmp, [&](void* x, size_t n) {
+			total += n;
+			if(total % (1024*1024) == 0) {
+				std::cout << ".";
+			}
+		});
+		auto diff = (microsec_clock::universal_time() - start).total_microseconds();
+		auto s = diff / 1000000.0;
+		if(s == 0) s = 1;
+		cout << endl << Util::formatBytes(total) << ", " << s << " s, " << Util::formatBytes(total / s) << " b/s" << endl;
+	} catch(const std::exception& e) {
+		cout << "Error: " << e.what() << endl;
+	}
+
+	return 0;
+}
+
+
+