Frequent delays in multithreaded code

I am trying to make my simple C # graphics library multithreaded. However, after introducing this code:

/* foreach (IAffector affector in affectorLookup.Values)
    affector.Update(timestep); */

taskManager.Value = timestep; taskManager.Start();

foreach (IAffector affector in affectorLookup.Values)
    taskManager.AddToQueue(affector.Update);
taskManager.StopWhenDone();
taskManager.Wait();

the simulation starts experiencing sharp lag spikes that seem to occur in TaskHandler.Run (I can't say for sure, because adding the previous code makes my code profiler ignore anything outside of TaskHandler.Run).

Task Manager:

public class TaskManager
{
    public delegate void MethodDel(float timestep);
    private Queue<MethodDel> queue;
    private List<TaskHandler> handlers;
    private float value;


    public float Value
    {
        get
        {
            return value;
        }
        set
        {
            this.value = value;
        }
    }


    public TaskManager()
    {
        this.queue = new Queue<MethodDel>();
        this.handlers = new List<TaskHandler>(System.Environment.ProcessorCount);

        for (int t = 0; t < this.handlers.Capacity; ++t)
            this.handlers.Add(new TaskHandler(this));

        this.value = 0;
    }


    public void Start()
    {
        foreach (var handler in handlers)
            handler.Start();
    }


    public void Stop()
    {
        lock (queue)
            queue.Clear();

        foreach (var handler in handlers)
            handler.StopWhenDone();
    }


    public void StopWhenDone()
    {
        foreach (var handler in handlers)
            handler.StopWhenDone();
    }


    public void AddToQueue(MethodDel method)
    {
        lock (queue)
            queue.Enqueue(method);
    }


    public bool GetFromQueue(out MethodDel method)
    {
        lock (queue)
        {
            if (queue.Count == 0) { method = null; return false; }

            method = queue.Dequeue();
            return true;
        }
    }


    public int GetQueueCount()
    {
        return queue.Count;
    }

    internal void Wait()
    {
        // Have to wait for them one at a time because the main thread is STA.

        WaitHandle[] waitHandles = new WaitHandle[1];
        // for (int t = 0; t < handlers.Count; ++t)
            // waitHandles[t] = handlers[t].WaitHandle;

        // WaitHandle.WaitAll(waitHandles);
        for (int t = 0; t < handlers.Count; ++t)
        { waitHandles[0] = handlers[t].WaitHandle; WaitHandle.WaitAll(waitHandles); }
    }
}

And the task handler:

public class TaskHandler
{
    private TaskManager manager;
    private Thread thread;
    private bool stopWhenDone;
    private ManualResetEvent waitHandle;


    public ManualResetEvent WaitHandle
    {
        get
        {
            return waitHandle;
        }
    }


    public TaskHandler(TaskManager manager)
    {
        this.manager = manager;
    }


    public void Start()
    {
        waitHandle = new ManualResetEvent(false);

        stopWhenDone = false;

        thread = new Thread(Run);
        thread.IsBackground = true;
        thread.SetApartmentState(ApartmentState.MTA);
        thread.Start();
    }


    public void StopWhenDone()
    {
        this.stopWhenDone = true;
    }

    // Possible source of slowdown
    private void Run()
    {
        TaskManager.MethodDel curMethod;
        while (!stopWhenDone || manager.GetQueueCount() > 0)
        {
            if (manager.GetFromQueue(out curMethod))
            {
                curMethod(manager.Value);
            }
        }
        waitHandle.Set();
    }
}