MVC 3 using custom httphandler with a different extension

No need to add httphandler. You must do this in asp.net mvc through a controller Example:

public class PictureController : Controller
{
    public FileResult GetImage(int pictureID)
    {
        byte[] fileContents = null;
        //Get the file here.
        return File(fileContents, "image/jpeg");
    }
}

in global.asax you can define

routes.MapRoute("Picture-GetImage", "picture/{pictureID}",
new { controller = "Picture", action = "GetImage" }

You can also use the System.Web.Helpers.WebImage helper or do it manually, for example:

public static byte[] ProcessCropResizeImage(string imageurl, Size outputSize)
{
    if (File.Exists(imageurl))
    {
        MemoryStream result = new MemoryStream();
        ImageCodecInfo codec = ImageCodecInfo.GetImageEncoders().FirstOrDefault(m => m.MimeType == "image/jpeg");
        if (codec == null)
            throw new ArgumentException(string.Format("Unsupported mimeType specified for encoding ({0})", "image/jpeg"), "encodingMimeType");
        EncoderParameters encoderParams = new EncoderParameters(1);
        encoderParams.Param[0] = new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, 85L);
        using (FileStream fs = File.OpenRead(imageurl))
        {
            using (Image image = Image.FromStream(fs))
            {
                using (Bitmap b = new Bitmap(outputSize.Width, outputSize.Height))
                {
                    using (Graphics g = Graphics.FromImage((Image)b))
                    {
                        g.CompositingQuality = CompositingQuality.HighQuality;
                        g.SmoothingMode = SmoothingMode.HighQuality;
                        g.InterpolationMode = InterpolationMode.HighQualityBicubic;
                        g.DrawImage(image, 0, 0, outputSize.Width, outputSize.Height);
                        g.DrawImage(image, outputSize.Width, outputSize.Height);
                    }
                    b.Save(result, codec, encoderParams);
                }
            }
        }

        return result.GetBuffer();
    }
    return null;
}