How to determine if the mouse moves when mouseup starts?

I tried to realize the panorama of the page by dragging it. In my implementation, the page moves for a while after the user releases the mouse button, for example, dragging and dropping maps in Google Maps. Now I would like to prevent this effect when the mouse no longer moves, when the user releases the button. The problem is that I just can't figure out how to determine if the mouse really moves when the event fires mouseup.

At the moment, I tried to solve this problem by calculating the drag and drop speed, and then compare the speed with the previously estimated "sensitivity", which works most of the time, but sometimes it fails.

Simplified example in jsFiddle . When you play the violin, use the middle button in FF, the divgabble div “sticks” to the left button.

pseudo code:

AniMove = function (doc, element, sensitivity, panspeed, duration) {
    var mouseDown = function (e) {
            sTime = new Date();
            originalX = mouseX = e.clientX;
            originalY = mouseY = e.clientY;
            /* addEventListeners mousemove & mouseup for document */
            return;
        },
        mouseMove = function (e) {
            /* Setting new position for #square + new mouseX & Y */
            return;
        },
        mouseUp = function () {
            var dc = 1;
                /* removeEventListeners mousemove & mouseup */
                eTime = new Date();
                vX = Math.round((50 * panspeed) * (originalX - mouseX) / (eTime - sTime));
                vY = Math.round((50 * panspeed) * (originalY - mouseY) / (eTime - sTime));

            // Check whether mouse is moving or not,
            // now checking the speed against sensitivity, which is not reliable

                if (Math.abs(vX) < sensitivity){vX = 0;}
                if (Math.abs(vY) < sensitivity){vY = 0;}

                for (n = 0; n < dur; n++, dc += 0.001) {
                    vX = Math.round(vX  * dec / dc);
                    vY = Math.round(vY * dec / dc);
                    delay[n] = setTimeout(endDrag(n, vX, vY), n * 50);
                }
                return;
            },
            endDrag = function (n, vX, vY) {
                /* Setting new position for #square */
                return;
            },
            dec = 1 - 120 / duration;
    panspeed *= -0.01;
    duration /= 50;
    element.addEventListener('mousedown', mouseDown, false);    
}
drag = new AniMove(document, document.getElementById('square'), 20, 100, 500);  

So, is it even possible to detect whether the mouse is moving or not in this situation? Maybe I need a different approach for this task?