linuxdcpp-team team mailing list archive

[noreply@xxxxxxxxxxxxx]

------------------------------------------------------------
revno: 2777
committer: Jacek Sieka <arnetheduck@xxxxxxxxx>
branch nick: dcplusplus
timestamp: Sat 2011-12-31 16:13:51 +0100
message:
  Mapped reader
modified:
  dcpp/FileReader.cpp
  dcpp/FileReader.h
  utils/xsum.cpp


--
lp:dcplusplus
https://code.launchpad.net/~dcplusplus-team/dcplusplus/trunk

Your team Dcplusplus-team is subscribed to branch lp:dcplusplus.
To unsubscribe from this branch go to https://code.launchpad.net/~dcplusplus-team/dcplusplus/trunk/+edit-subscription

=== modified file 'dcpp/FileReader.cpp'
--- dcpp/FileReader.cpp	2011-12-31 11:50:24 +0000
+++ dcpp/FileReader.cpp	2011-12-31 15:13:51 +0000
@@ -36,28 +36,34 @@
 
 /** Read entire file, never returns READ_FAILED */
 size_t FileReader::readCached(const string& file, const DataCallback& callback) {
-	auto buf = getBuffer(512);
+	buffer.resize(getBlockSize(0));
 
+	auto buf = &buffer[0];
 	File f(file, File::READ, File::OPEN | File::SHARED);
 
 	size_t total = 0;
-	size_t n = buf.second;
-	while(f.read(buf.first, n) > 0) {
-		callback(buf.first, n);
+	size_t n = buffer.size();
+	while(f.read(buf, n) > 0) {
+		callback(buf, n);
 		total += n;
-		n = buf.second;
+		n = buffer.size();
 	}
 
 	return total;
 }
 
-pair<void*, size_t> FileReader::getBuffer(size_t alignment) {
-	auto block = (((blockSize == 0 ? DEFAULT_BLOCK_SIZE : blockSize) + alignment - 1) / alignment) * alignment;
-
-	buffer.resize(block * 2 + alignment); // Prepare for worst case alignment
-
-	auto start = reinterpret_cast<void*>(((reinterpret_cast<size_t>(&buffer[0]) + alignment - 1) / alignment) * alignment);
-	return make_pair(start, block);
+size_t FileReader::getBlockSize(size_t alignment) {
+	auto block = blockSize < DEFAULT_BLOCK_SIZE ? DEFAULT_BLOCK_SIZE : blockSize;
+	if(alignment > 0) {
+		block = ((block + alignment - 1) / alignment) * alignment;
+	}
+
+	return block;
+}
+
+void* FileReader::align(void *buf, size_t alignment) {
+	return alignment == 0 ? buf
+		: reinterpret_cast<void*>(((reinterpret_cast<size_t>(buf) + alignment - 1) / alignment) * alignment);
 }
 
 #ifdef _WIN32
@@ -86,64 +92,66 @@
 
 	if (tmp == INVALID_HANDLE_VALUE) {
 		dcdebug("Failed to open unbuffered file: %s\n", Util::translateError(::GetLastError()).c_str());
-		return false;
+		return READ_FAILED;
 	}
 
 	Handle h(tmp);
 
-	auto buf = getBuffer(sector);
-	DWORD bufSize = static_cast<DWORD>(buf.second);
+	DWORD bufSize = static_cast<DWORD>(getBlockSize(sector));
+	buffer.resize(bufSize * 2 + sector);
+
+	auto buf = align(&buffer[0], sector);
 
 	DWORD hn = 0;
 	DWORD rn = 0;
-	uint8_t* hbuf = static_cast<uint8_t*>(buf.first) + bufSize;
-	uint8_t* rbuf = static_cast<uint8_t*>(buf.first);
+	uint8_t* hbuf = static_cast<uint8_t*>(buf) + bufSize;
+	uint8_t* rbuf = static_cast<uint8_t*>(buf);
 	OVERLAPPED over = { 0 };
 
 	// Read the first block
-	auto res = ::ReadFile(h, hbuf, buf.second, &hn, &over);
-
-	if(!res) {
+	auto res = ::ReadFile(h, hbuf, bufSize, NULL, &over);
+	auto err = ::GetLastError();
+
+	if(!res && err != ERROR_IO_PENDING) {
+		if(err != ERROR_HANDLE_EOF) {
+			dcdebug("First overlapped read failed: %s\n", Util::translateError(::GetLastError()).c_str());
+			return READ_FAILED;
+		}
+	}
+
+	// Finish the read and see how it went
+	if(!GetOverlappedResult(h, &over, &hn, TRUE)) {
+		err = ::GetLastError();
+		if(err != ERROR_HANDLE_EOF) {
+			dcdebug("First overlapped read failed: %s\n", Util::translateError(::GetLastError()).c_str());
+			return READ_FAILED;
+		}
+	}
+	over.Offset = hn;
+
+	for (; hn == bufSize;) {
+		// Start a new overlapped read
+		res = ::ReadFile(h, rbuf, bufSize, NULL, &over);
 		auto err = ::GetLastError();
-		if (err == ERROR_HANDLE_EOF) {
-			hn = 0;
-		} else if(err == ERROR_IO_PENDING) {
-			// Finish the read and see how it went
-			if(!GetOverlappedResult(h, &over, &hn, TRUE)) {
-				if (::GetLastError() == ERROR_HANDLE_EOF) {
-					hn = 0;
-				} else {
-					dcdebug("First overlapped read failed: %s\n", Util::translateError(::GetLastError()).c_str());
-					return READ_FAILED;
+
+		// Process the previously read data
+		callback(hbuf, hn);
+
+		if (!res && err != ERROR_IO_PENDING) {
+			if(err != ERROR_HANDLE_EOF) {
+				throw FileException(Util::translateError(err));
+			}
+
+			rn = 0;
+		} else {
+			// Finish the new read
+			if (!GetOverlappedResult(h, &over, &rn, TRUE)) {
+				err = ::GetLastError();
+				if(err != ERROR_HANDLE_EOF) {
+					throw FileException(Util::translateError(err));
 				}
-			}
-		}
-	}
-
-	over.Offset = hn;
-
-	for (; hn > 0;) {
-		// Last read returned some bytes, start a new overlapped read
-		res = ::ReadFile(h, rbuf, buf.second, &rn, &over);
-
-		callback(hbuf, hn);
-
-		if (!res) {
-			auto err = ::GetLastError();
-			if(err == ERROR_HANDLE_EOF) {
+
 				rn = 0;
-			} else if(err == ERROR_IO_PENDING) {
-				// Finish the read
-				if (!GetOverlappedResult(h, &over, &rn, TRUE)) {
-					err = ::GetLastError();
-					if(err != ERROR_HANDLE_EOF) {
-						throw FileException(Util::translateError(err));
-					}
-
-					rn = 0;
-				}
-			} else {
-				throw FileException(Util::translateError(::GetLastError()));
 			}
 		}
 
@@ -153,11 +161,64 @@
 		swap(rn, hn);
 	}
 
+	if(hn != 0) {
+		// Process leftovers
+		callback(hbuf, hn);
+	}
+
 	return *((uint64_t*)&over.Offset);
 }
 
 size_t FileReader::readMapped(const string& file, const DataCallback& callback) {
-	return READ_FAILED;
+	auto tfile = Text::toT(file);
+
+	auto tmp = ::CreateFile(tfile.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING,
+		FILE_FLAG_SEQUENTIAL_SCAN, NULL);
+
+	if (tmp == INVALID_HANDLE_VALUE) {
+		dcdebug("Failed to open unbuffered file: %s\n", Util::translateError(::GetLastError()).c_str());
+		return READ_FAILED;
+	}
+
+	Handle h(tmp);
+
+	LARGE_INTEGER size = { 0 };
+	if(!::GetFileSizeEx(h, &size)) {
+		dcdebug("Couldn't get file size: %s\n", Util::translateError(::GetLastError()).c_str());
+		return READ_FAILED;
+	}
+
+	if(!(tmp = ::CreateFileMapping(h, NULL, PAGE_READONLY, 0, 0, NULL))) {
+		dcdebug("Couldn't create file mapping: %s\n", Util::translateError(::GetLastError()).c_str());
+		return READ_FAILED;
+	}
+
+	Handle hmap(tmp);
+
+	SYSTEM_INFO si = { 0 };
+	::GetSystemInfo(&si);
+
+	auto blockSize = getBlockSize(si.dwPageSize);
+
+	LARGE_INTEGER total = { 0 };
+	while(size.QuadPart > 0) {
+		auto n = min(size.QuadPart, (int64_t)blockSize);
+		auto p = ::MapViewOfFile(hmap, FILE_MAP_READ, total.HighPart, total.LowPart, static_cast<DWORD>(n));
+		if(!p) {
+			throw FileException(Util::translateError(::GetLastError()));
+		}
+
+		callback(p, n);
+
+		if(!::UnmapViewOfFile(p)) {
+			throw FileException(Util::translateError(::GetLastError()));
+		}
+
+		size.QuadPart -= n;
+		total.QuadPart += n;
+	}
+
+	return total.QuadPart;
 }
 
 #else

=== modified file 'dcpp/FileReader.h'
--- dcpp/FileReader.h	2011-12-31 11:50:24 +0000
+++ dcpp/FileReader.h	2011-12-31 15:13:51 +0000
@@ -71,7 +71,8 @@
 	vector<uint8_t> buffer;
 
 	/** Return an aligned buffer which is at least twice the size of ret.second */
-	pair<void*, size_t> getBuffer(size_t alignment);
+	size_t getBlockSize(size_t alignment);
+	void* align(void* buf, size_t alignment);
 
 	size_t readDirect(const string& file, const DataCallback& callback);
 	size_t readMapped(const string& file, const DataCallback& callback);

=== modified file 'utils/xsum.cpp'
--- utils/xsum.cpp	2011-12-31 11:50:24 +0000
+++ utils/xsum.cpp	2011-12-31 15:13:51 +0000
@@ -3,6 +3,8 @@
 #include <dcpp/stdinc.h>
 #include <dcpp/FileReader.h>
 #include <dcpp/Util.h>
+#include <dcpp/MerkleTree.h>
+#include <stdlib.h>
 
 #include <boost/date_time/posix_time/ptime.hpp>
 using namespace boost::posix_time;
@@ -12,28 +14,33 @@
 
 int main(int argc, char** argv)
 {
-	if(argc != 2) {
+	if(argc < 2) {
 		cout << "You need to supply a file name" << endl;
 		return 1;
 	}
 
 	char x[_MAX_PATH] = { 0 };
-	char * tmp;
-	if(!(tmp = _fullpath(x, argv[1], _MAX_PATH))) {
+	if(!_fullpath(x, argv[1], _MAX_PATH)) {
 		cout << "Can't get full path" << endl;
 		return 1;
 	}
+
 	try {
 		auto start = microsec_clock::universal_time();
-		FileReader fr(true, 0);
+		FileReader fr(argc == 2 ? true : argv[2][0] == '0', 0);
 
+		TigerTree tt;
 		size_t total = 0;
-		fr.read(tmp, [&](void* x, size_t n) {
+		fr.read(x, [&](void* x, size_t n) {
+			tt.update(x, n);
 			total += n;
 			if(total % (1024*1024) == 0) {
 				std::cout << ".";
 			}
 		});
+
+		cout << endl << Encoder::toBase32(tt.finalize(), TigerTree::BYTES);
+
 		auto diff = (microsec_clock::universal_time() - start).total_microseconds();
 		auto s = diff / 1000000.0;
 		if(s == 0) s = 1;