← Back to team overview

beeseek-devs team mailing list archive

[Branch ~beeseek-devs/beeseek/trunk] Rev 201: Add a separate package for decoders.

 

------------------------------------------------------------
revno: 201
committer: Andrea Corbellini <andrea.corbellini@xxxxxxxxxxx>
branch nick: trunk
timestamp: Mon 2009-01-19 20:46:13 +0100
message:
  Add a separate package for decoders.
  
  HTTPApplication objects not use an external decoder package for every 
  kind of transfer (chunked, fixed-length, ...). This removes the 
  reference cycles and makes the code a bit faster.
  Also it is easier for developers to expand and hack classes that may 
  require special decoders.
added:
  beeseek/decoders/
  beeseek/decoders/__init__.py
  beeseek/decoders/base.py
  beeseek/decoders/chunks.py
modified:
  beeseek/network/http.py
  beeseek/tests/network.py
    ------------------------------------------------------------
    revno: 200.1.7
    committer: Andrea Corbellini <andrea.corbellini@xxxxxxxxxxx>
    branch nick: decoders
    timestamp: Mon 2009-01-19 20:44:31 +0100
    message:
      Remove unused attributes from __slots__.
    modified:
      beeseek/network/http.py
    ------------------------------------------------------------
    revno: 200.1.6
    committer: Andrea Corbellini <andrea.corbellini@xxxxxxxxxxx>
    branch nick: decoders
    timestamp: Mon 2009-01-19 20:38:41 +0100
    message:
      Update HTTPApplication objects code.
    modified:
      beeseek/network/http.py
      beeseek/tests/network.py
    ------------------------------------------------------------
    revno: 200.1.5
    committer: Andrea Corbellini <andrea.corbellini@xxxxxxxxxxx>
    branch nick: decoders
    timestamp: Mon 2009-01-19 20:38:19 +0100
    message:
      Add NullSocketDecoder.
    modified:
      beeseek/decoders/base.py
    ------------------------------------------------------------
    revno: 200.1.4
    committer: Andrea Corbellini <andrea.corbellini@xxxxxxxxxxx>
    branch nick: decoders
    timestamp: Mon 2009-01-19 20:38:06 +0100
    message:
      Add FixedLengthChunkDecoder.
    modified:
      beeseek/decoders/__init__.py
      beeseek/decoders/chunks.py
    ------------------------------------------------------------
    revno: 200.1.3
    committer: Andrea Corbellini <andrea.corbellini@xxxxxxxxxxx>
    branch nick: decoders
    timestamp: Mon 2009-01-19 18:14:01 +0100
    message:
      Add ChunksDecoder to decode HTTP chunks.
    added:
      beeseek/decoders/chunks.py
    ------------------------------------------------------------
    revno: 200.1.2
    committer: Andrea Corbellini <andrea.corbellini@xxxxxxxxxxx>
    branch nick: decoders
    timestamp: Mon 2009-01-19 17:54:34 +0100
    message:
      Add interfaces for decoders.
    added:
      beeseek/decoders/base.py
    modified:
      beeseek/decoders/__init__.py
    ------------------------------------------------------------
    revno: 200.1.1
    committer: Andrea Corbellini <andrea.corbellini@xxxxxxxxxxx>
    branch nick: decoders
    timestamp: Sun 2009-01-18 13:43:39 +0100
    message:
      Add decoders package.
    added:
      beeseek/decoders/
      beeseek/decoders/__init__.py

=== added directory 'beeseek/decoders'
=== added file 'beeseek/decoders/__init__.py'
--- a/beeseek/decoders/__init__.py	1970-01-01 00:00:00 +0000
+++ b/beeseek/decoders/__init__.py	2009-01-19 19:38:06 +0000
@@ -0,0 +1,19 @@
+#coding=UTF-8
+
+# Copyright (C) 2007-2009  BeeSeek Developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from beeseek.decoders.base import *
+from beeseek.decoders.chunks import *

=== added file 'beeseek/decoders/base.py'
--- a/beeseek/decoders/base.py	1970-01-01 00:00:00 +0000
+++ b/beeseek/decoders/base.py	2009-01-19 19:38:19 +0000
@@ -0,0 +1,92 @@
+#coding=UTF-8
+
+# Copyright (C) 2007-2009  BeeSeek Developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from beeseek.interfaces import Interface, implements
+
+__all__ = 'IDecoder', 'IStreamDecoder', 'ISocketDecoder', 'NullSocketDecoder'
+
+
+class IDecoder(Interface):
+    """Base interface for all decoders."""
+
+
+class IStreamDecoder(IDecoder):
+    """Decode the given data."""
+
+    def decode(self, data):
+        """Decode the given data.
+
+        :param data: The encoded string.
+        :type data: str
+        :return: A string with the decoded data.
+        """
+
+
+class ISocketDecoder(IDecoder):
+    """Decode data from a socket.
+
+    Get the string from a socket and decode them.
+    """
+
+    def recv(self, fp, size=-1):
+        """Read and decode at most the specified bytes.
+
+        :param fp: The socket.
+        :param size: The maximum number of bytes to read.
+        :type size: int
+        """
+
+    def read(self, fp, size=-1):
+        """Read and decode the specified bytes or the whole stream.
+
+        :param fp: The socket.
+        :param size: The number of bytes to read. If lesser than 0, read the
+                     whole stream.
+        :type size: int
+        """
+
+    def readline(self, fp, size=-1):
+        """Read and decode the next line.
+
+        :param fp: The socket.
+        :param size: The maximum number of bytes to read.
+        :type size: int
+        """
+
+    def iter_raw_body(self, fp):
+        pass
+
+
+class NullSocketDecoder(object):
+
+    implements(ISocketDecoder)
+
+    def recv(self, fp, size=-1):
+        return fp.raw_recv(size)
+
+    def read(self, fp, size=-1):
+        return fp.raw_read(size)
+
+    def readline(self, fp, size=-1):
+        return fp.raw_readline(size)
+
+    def iter_raw_body(self, fp):
+        while True:
+            data = fp.recv(4096)
+            if not data:
+                return
+            yield data

=== added file 'beeseek/decoders/chunks.py'
--- a/beeseek/decoders/chunks.py	1970-01-01 00:00:00 +0000
+++ b/beeseek/decoders/chunks.py	2009-01-19 19:38:06 +0000
@@ -0,0 +1,171 @@
+#coding=UTF-8
+
+# Copyright (C) 2007-2009  BeeSeek Developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from beeseek.interfaces import implements
+from beeseek.decoders import ISocketDecoder
+
+__all__ = 'ChunksDecoder', 'FixedLengthChunkDecoder'
+
+
+class ChunksDecoder(object):
+
+    __slots__ = '_chunklen'
+    implements(ISocketDecoder)
+
+    def __init__(self):
+        self._chunklen = 0
+
+    def _read_chunksize(self, fp):
+        line = fp.raw_readline()
+        if line == '\r\n':
+            line = fp.raw_readline()
+        return int(line, 16)
+
+    def _last_chunk(self, fp):
+        fp.raw_readline()
+        self._chunklen = -1
+
+
+    def recv(self, fp, size=-1):
+        chunklen = self.chunklen
+        if chunklen == 0:
+            chunklen = self._read_chunksize(fp)
+            if chunklen == 0:
+                self._last_chunk(fp)
+                return ''
+        elif chunklen < 0:
+            return ''
+
+        if size > chunklen or size < 0:
+            size = chunklen
+        data = fp.raw_recv(size)
+        self._chunklen = chunklen - len(data)
+        return data
+
+    def read(self, fp, size=-1):
+        data = ''
+        chunklen = self._chunklen
+        if chunklen == 0:
+            chunklen = self._read_chunksize(fp)
+            if chunklen == 0:
+                self._last_chunk(fp)
+                return ''
+        elif chunklen < 0:
+            return ''
+
+        if size < 0:
+            while True:
+                data += fp.raw_read(chunklen)
+                chunklen = self._read_chunksize(fp)
+                if chunklen == 0:
+                    self._last_chunk(fp)
+                    return data
+        else:
+            while True:
+                if chunklen < size:
+                    piece = fp.raw_read(chunklen)
+                    data += piece
+                    chunklen = self._read_chunksize(fp)
+                    if chunklen == 0:
+                        self._last_chunk(fp)
+                        return data
+                    size -= len(piece)
+                else:
+                    data += fp.raw_read(size)
+                    self._chunklen = chunklen - size
+                    return data
+
+    def readline(self, fp, size=-1):
+        data = piece = ''
+        chunklen = self._chunklen
+        if chunklen == 0:
+            chunklen = self._read_chunksize(fp)
+            if chunklen == 0:
+                self._last_chunk(fp)
+                return ''
+        elif chunklen < 0:
+            return ''
+
+        if size < 0:
+            while True:
+                piece = fp.raw_readline(chunklen)
+                data += piece
+                if piece.endswith('\n'):
+                    self._chunklen = chunklen - len(piece)
+                    return data
+                chunklen = self._read_chunksize(fp)
+                if chunklen == 0:
+                    self._last_chunk(fp)
+                    return data
+        else:
+            while True:
+                if chunklen < size:
+                    piece = fp.raw_readline(chunklen)
+                    data += piece
+                    if piece.endswith('\n'):
+                        self._chunklen = chunklen - len(piece)
+                        return data
+                    chunklen = self._read_chunksize(fp)
+                    if chunklen == 0:
+                        self._last_chunk(fp)
+                        return data
+                    size -= len(piece)
+                else:
+                    piece = fp.raw_readline(size)
+                    data += piece
+                    self._chunklen = chunklen - len(piece)
+                    return data
+
+    def iter_raw_body(self, fp):
+        chunklen = self._chunklen
+        if chunklen < 0:
+            return
+        elif chunklen:
+            yield fp.raw_read(chunklen)
+
+        while True:
+            chunklen = self._read_chunksize(fp)
+            if chunklen == 0:
+                yield '0\r\n\r\n'
+                self._chunklen = -1
+                return
+            yield '%x\r\n%s' % (chunklen, fp.raw_read(chunklen))
+
+        self._last_chunk(fp)
+
+
+class FixedLengthChunkDecoder(ChunksDecoder):
+
+    def __init__(self, size):
+        self._chunklen = size
+
+    def _read_chunksize(self, fp):
+        return 0
+
+    def _last_chunk(self, fp):
+        self._chunklen = -1
+
+    def iter_raw_body(self, fp):
+        chunklen = self._chunklen
+        if chunklen <= 0:
+            return
+
+        while chunklen > 0:
+            yield fp.raw_read(min(chunklen, 4096))
+            chunklen -= 4096
+
+        self._last_chunk(fp)

=== modified file 'beeseek/network/http.py'
--- a/beeseek/network/http.py	2009-01-17 16:41:25 +0000
+++ b/beeseek/network/http.py	2009-01-19 19:44:31 +0000
@@ -19,6 +19,8 @@
 from beeseek.utils import CaseInsensitiveDict
 from beeseek.network import (IClientApplication, IServerApplication,
                              IApplicationProtocol, BaseApplication)
+from beeseek.decoders import (NullSocketDecoder, ChunksDecoder,
+                              FixedLengthChunkDecoder)
 
 __all__ = 'HTTPApplication', 'HTTPClientApplication', 'HTTPServerApplication'
 
@@ -79,8 +81,8 @@
     and untested but may work in some cases.
     """
 
-    __slots__ = ('recv', 'read', 'readline', 'write', 'iter_raw_body',
-                 'end_request', 'end_response', 'will_close', '_contentlen')
+    __slots__ = ('write', 'end_request', 'end_response', 'will_close',
+                 '_decoder')
     implements(IHTTPApplication, isbase=True)
 
 
@@ -159,142 +161,17 @@
             name, value = line.split(':', 1)
             yield name, value.strip()
 
-    # * Read fixed length body
-
-    def _recv_fixedlen(self, size=-1):
-        if size > self._contentlen or size < 0:
-            size = self._contentlen
-        data = self.raw_recv(size)
-        self._contentlen -= len(data)
-        return data
-
-    def _read_fixedlen(self, size=-1):
-        if size > self._contentlen or size < 0:
-            size = self._contentlen
-        data = self.raw_read(size)
-        self._contentlen -= len(data)
-        return data
-
-    def _readline_fixedlen(self, size=-1):
-        if size > self._contentlen or size < 0:
-            size = self._contentlen
-        data = self.raw_readline(size)
-        self._contentlen -= len(data)
-        return data
-
-    # * Read chunks
-
-    def _read_chunksize(self):
-        line = self.raw_readline()
-        if line == '\r\n':
-            line = self.raw_readline()
-            if line == '\r\n':
-                return 0
-        return int(line, 16)
-
-    def _recv_chunk(self, size=-1):
-        chunklen = self._contentlen
-        if chunklen == 0:
-            chunklen = self._read_chunksize()
-            if chunklen == 0:
-                self.raw_readline()
-                self._contentlen = -1
-                return ''
-        elif chunklen < 0:
-            return ''
-
-        if size > chunklen or size < 0:
-            size = chunklen
-        data = self.raw_recv(size)
-        self._contentlen = chunklen - len(data)
-        return data
-
-    def _read_chunk(self, size=-1):
-        data = ''
-        chunklen = self._contentlen
-        if chunklen < 0:
-            return ''
-        if size < 0:
-            while True:
-                data += self.raw_read(chunklen)
-                chunklen = self._read_chunksize()
-                if chunklen == 0:
-                    self.raw_readline()
-                    self._contentlen = -1
-                    return data
-        else:
-            while True:
-                if chunklen < size:
-                    piece = self.raw_read(chunklen)
-                    data += piece
-                    chunklen = self._read_chunksize()
-                    if chunklen == 0:
-                        self.raw_readline()
-                        self._contentlen = -1
-                        return data
-                    size -= len(piece)
-                else:
-                    data += self.raw_read(size)
-                    self._contentlen = chunklen - size
-                    return data
-
-    def _readline_chunk(self, size=-1):
-        data = piece = ''
-        chunklen = self._contentlen
-        if chunklen < 0:
-            return ''
-        if size < 0:
-            while True:
-                piece = self.raw_readline(chunklen)
-                data += piece
-                if piece.endswith('\n'):
-                    self._contentlen = chunklen - len(piece)
-                    return data
-                chunklen = self._read_chunksize()
-                if chunklen == 0:
-                    self.raw_readline()
-                    self._contentlen = -1
-                    return data
-        else:
-            while True:
-                if chunklen < size:
-                    piece = self.raw_readline(chunklen)
-                    data += piece
-                    if piece.endswith('\n'):
-                        self._contentlen = chunklen - len(piece)
-                        return data
-                    chunklen = self._read_chunksize()
-                    if chunklen == 0:
-                        self.raw_readline()
-                        self._contentlen = -1
-                        return data
-                    size -= len(piece)
-                else:
-                    piece = self.raw_readline(size)
-                    data += piece
-                    self._contentlen = chunklen - len(piece)
-                    return data
-
-    def _iter_raw_standard_body(self):
-        while True:
-            data = self.recv(131072)
-            if not data:
-                return
-            yield data
-
-    def _iter_raw_chunked_body(self):
-        chunklen = self._contentlen
-        if chunklen < 0:
-            return
-        elif chunklen:
-            yield self.raw_read(self._contentlen)
-        while True:
-            chunklen = self._read_chunksize()
-            if chunklen == 0:
-                yield '0\r\n\r\n'
-                self._contentlen = -1
-                return
-            yield '%x\r\n%s' % (chunklen, self.raw_read(chunklen))
+    def recv(self, size=-1):
+        return self._decoder.recv(self, size)
+
+    def read(self, size=-1):
+        return self._decoder.read(self, size)
+
+    def readline(self, size=-1):
+        return self._decoder.readline(self, size)
+
+    def iter_raw_body(self):
+        return self._decoder.iter_raw_body(self)
 
     # Methods to send parts of a request or a response
 
@@ -347,8 +224,8 @@
         self._end_standard_message()
 
     def _end_standard_message(self):
-        del (self.recv, self.read, self.readline, self.write,
-             self.iter_raw_body, self.end_response, self.end_request)
+        #del (self.recv, self.read, self.readline, self.write,
+        #     self.iter_raw_body, self.end_response, self.end_request)
         if self.will_close:
             self.close()
 
@@ -385,30 +262,19 @@
 
     def _set_read_methods(self, style, headers):
         if style == 0:
-            self.recv = self.read = self.readline = self._read_eof
-            self.iter_raw_body = self._iter_raw_standard_body
+            self._decoder = FixedLengthChunkDecoder(-1)
         elif style == 1:
-            self.recv = self.raw_recv
-            self.read = self.raw_read
-            self.readline = self.raw_readline
-            self.iter_raw_body = self._iter_raw_standard_body
+            self._decoder = NullSocketDecoder()
         elif style == 2:
-            self._contentlen = int(headers['Content-Length'])
-            self.recv = self._recv_fixedlen
-            self.read = self._read_fixedlen
-            self.readline = self._readline_fixedlen
-            self.iter_raw_body = self._iter_raw_standard_body
+            size = int(headers['Content-Length'])
+            self._decoder = FixedLengthChunkDecoder(size)
         else:
-            self._contentlen = 0
-            self.recv = self._recv_chunk
-            self.read = self._read_chunk
-            self.readline = self._readline_chunk
-            self.iter_raw_body = self._iter_raw_chunked_body
+            self._decoder = ChunksDecoder()
 
 
 class HTTPClientApplication(HTTPApplication):
 
-    __slots__ = 'statusline'
+    __slots__ = ()
     implements(IHTTPApplication, IClientApplication)
 
 

=== modified file 'beeseek/tests/network.py'
--- a/beeseek/tests/network.py	2009-01-17 16:41:25 +0000
+++ b/beeseek/tests/network.py	2009-01-19 19:38:41 +0000
@@ -111,7 +111,8 @@
         assert retval[0] == (peer.POST, page, peer.HTTP11)
         assert dict(retval[1].iteritems()) == headers
 
-        client.write(string)
+        client.write(string[:200])
+        client.write(string[200:])
         client.end_request()
         client.flush()
         assert peer.read() == string



--
BeeSeek mainline
https://code.launchpad.net/~beeseek-devs/beeseek/trunk

You are receiving this branch notification because you are subscribed to it.