A twisted HTTP server with an asynchronous response, where requests must wait until data becomes available or timeout

I am trying to write a simple http server that processes asynchronous requests that look in the data structure for a response or timeout:

• Request arrives
• So far, the time <timeout check responseCollector for the response (using requestId as the key)
• If the answer, return it
• If timeout, return a timeout message

I'm new to twisting, and I'm wondering what is the best way to make an asynchronous answer. I looked at some folded deferred documents and callLater , but it wasn’t clear to me exactly what I should do. Now I am running the lock method using deferToThread and waiting for the timeout to expire. I get a non-invoked error string for my deferred method:

Unhandled error in Deferred:
Traceback (most recent call last):
  File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/threading.py", line 497, in __bootstrap
    self.__bootstrap_inner()
  File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/threading.py", line 522, in __bootstrap_inner
    self.run()
  File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/threading.py", line 477, in run
    self.__target(*self.__args, **self.__kwargs)
--- <exception caught here> ---
  File "/System/Library/Frameworks/Python.framework/Versions/2.6/Extras/lib/python/twisted/python/threadpool.py", line 210, in _worker
    result = context.call(ctx, function, *args, **kwargs)
  File "/System/Library/Frameworks/Python.framework/Versions/2.6/Extras/lib/python/twisted/python/context.py", line 59, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/System/Library/Frameworks/Python.framework/Versions/2.6/Extras/lib/python/twisted/python/context.py", line 37, in callWithContext
    return func(*args,**kw)
exceptions.TypeError: 'str' object is not callable

Here is my code:

from twisted.web import server, resource
from twisted.internet import reactor, threads
import json
import time

connectedClients = {}
responseCollector = {}

# add fake data to the collector
class FakeData(resource.Resource):
    isLeaf = True

    def render_GET(self, request):
        request.setHeader("content-type", "application/json")
        if 'rid' in request.args and 'data' in request.args:
            rid = request.args['rid'][0]
            data = request.args['data'][0]
            responseCollector[str(rid)] = data
            return json.dumps(responseCollector)
        return "{}"

class RequestHandler(resource.Resource):
    isLeaf = True

    def render_GET(self, request):
        #request.setHeader("content-type", "application/json")

        if not ('rid' in request.args and and 'json' in request.args):
            return '{"success":"false","response":"invalid request"}'

        rid = request.args['rid'][0]
        json = request.args['id'][0]

        # TODO: Wait for data to show up in the responseCollector with same rid
        # as our request without blocking other requests OR timeout
        d = threads.deferToThread(self.blockingMethod(rid))
        d.addCallback(self.ret)
        d.addErrback(self.err)

    def blockingMethod(self,rid):
        timeout  = 5.0
        timeElapsed = 0.0
        while timeElapsed<timeout:
            if rid in responseCollector:
                return responseCollector[rid]
            else:
                timeElapsed+=0.01
                time.sleep(0.01)
        return "timeout"

    def ret(self, hdata):
        return hdata

    def err(self, failure):
        return failure

reactor.listenTCP(8080, server.Site(RequestHandler()))
reactor.listenTCP(9080, server.Site(FakeData()))
reactor.run()

Make a query (doesn't return anything useful at the moment):

http://localhost:8080/?rid=1234&json={%22foo%22:%22bar%22}

Add some fake data to use with the query:

http://localhost:9080/?rid=1234&data=foo

Update with production version

from twisted.web import server, resource
from twisted.internet import reactor, threads
import json
import time

connectedClients = {}
responseCollector = {}

# add fake data to the collector
class FakeData(resource.Resource):
    isLeaf = True

    def render_GET(self, request):
        request.setHeader("content-type", "application/json")
        if 'rid' in request.args and 'data' in request.args:
            rid = request.args['rid'][0]
            data = request.args['data'][0]
            responseCollector[str(rid)] = data
            return json.dumps(responseCollector)
        return "{}"

class RequestHandler(resource.Resource):
    isLeaf = True

    def render_GET(self, request):

        if not ('rid' in request.args and 'data' in request.args):
            return '{"success":"false","response":"invalid request"}'

        rid = request.args['rid'][0]
        json = request.args['data'][0]

        # TODO: Wait for data to show up in the responseCollector with same rid
        # as our request without blocking other requests OR timeout
        d = threads.deferToThread(self.blockingMethod,rid)
        d.addCallback(self.ret, request)
        d.addErrback(self.err)

        return server.NOT_DONE_YET

    def blockingMethod(self,rid):
        timeout  = 5.0
        timeElapsed = 0.0
        while timeElapsed<timeout:
            if rid in responseCollector:
                return responseCollector[rid]
            else:
                timeElapsed+=0.01
                time.sleep(0.01)
        return "timeout"

    def ret(self, result, request):
        request.write(result)
        request.finish()

    def err(self, failure):
        return failure

reactor.listenTCP(8080, server.Site(RequestHandler()))
reactor.listenTCP(9080, server.Site(FakeData()))
reactor.run()