OutOfMemory exception when using large thread in a C # application

I am writing a C # application that cycles through articles from a local copy of the Wikipedia database. I use a bunch of regular expressions to find the information I need in these articles, start the stream to get an image for each article, save the information and go to the next article.

I need to use a proxy list to download these images so as not to be banned from Google. Since the proxy server can be slow, I use streams for parallel loading.

If I do not use streams, the application works correctly, but it takes some time to get all the information.

If I use threads, the application runs until it uses about 500 threads, and then I get an OutOfMemory exception.

The fact is that it uses only ~ 300Mo of RAM, so it does not use all of the memory, either availlable (8Go) shared memory, or the memory allocated for one 32-bit application.

Is there a thread limit for each application?

EDIT:

Here is the code to download the poster (starts with getPosterAsc ()).

    string ddlValue = "";
    private void tryDownload(object obj)
    {
        WebClient webClientProxy = new WebClient();
        Tuple<WebProxy, int> proxy = (Tuple<WebProxy, int>)((object[])obj)[0];
        if (proxy != null)
            webClientProxy.Proxy = proxy.Item1;
        try
        {
            ddlValue = webClientProxy.DownloadString((string)((object[])obj)[1]);
        }
        catch (Exception ex) { 
            ddlValue = "";

            Console.WriteLine("trydownload:" + ex.Message);
        }

        webClientProxy.Dispose();
    }

    public void getPoster(object options = null)
    {
        if (options == null)
            options = new object[2] { toSave, false };
        if (!AppVar.debugMode && AppVar.getImages && this.getImage)
        {
            if (this.original_name != "" && !this.ambName && this.suitable)
            {
                Log.CountImgInc();

                MatchCollection MatchList;
                string basic_options = "";
                string value = "";
                WebClient webClient = new WebClient();
                Regex reg;
                bool found = false;

                if (original_name.Split(' ').Length > 1) image_options = "";

                if (!found)
                {
                    bool succes = false;
                    int countTry = 0;
                    while (!succes)
                    {
                        Tuple<WebProxy, int> proxy = null;
                        if (countTry != 5)
                            proxy = Proxy.getProxy();

                        try
                        {
                            Thread t = new Thread(tryDownload);
                            if (!(bool)((object[])options)[1])
                                t.Start(new object[] { proxy, @"http://www.google.com/search?as_st=y&tbm=isch&as_q=" + image_options + "+" + basic_options + "+" + image_options_before + "%22" + simplify(original_name) + "%22+" + " OR %22" + original_name + "%22+" + image_options_after + this.image_format });
                            else
                                t.Start(new object[] { proxy, @"http://www.google.com/search?as_st=y&tbm=isch&as_q=" + image_options + "+" + basic_options + "+" + image_options_before + "%22" + simplify(original_name) + "%22+" + " OR %22" + original_name + "%22+" + image_options_after + "&biw=1218&bih=927&tbs=isz:ex,iszw:758,iszh:140,ift:jpg&tbm=isch&source=lnt&sa=X&ei=kuG7T6qaOYKr-gafsOHNCg&ved=0CIwBEKcFKAE" });
                            if (!t.Join(40000))
                            {
                                Proxy.badProxy(proxy.Item1.Address.Host, proxy.Item1.Address.Port);
                                continue;
                            }
                            else
                            {
                                value = ddlValue;
                                if (value != "")
                                    succes = true;
                                else
                                    Proxy.badProxy(proxy.Item1.Address.Host, proxy.Item1.Address.Port);
                            }
                        }
                        catch (Exception ex)
                        {
                            if (proxy != null)
                                Proxy.badProxy(proxy.Item1.Address.Host, proxy.Item1.Address.Port);
                        }
                        countTry++;
                    }

                    reg = new Regex(@"imgurl\=(.*?)&amp;imgrefurl", RegexOptions.IgnoreCase);
                    MatchList = reg.Matches(value);
                    if (MatchList.Count > 0)
                    {
                        bool foundgg = false;
                        int j = 0;
                        while (!foundgg && MatchList.Count > j)
                        {
                            if (MatchList[j].Groups[1].Value.Substring(MatchList[j].Groups[1].Value.Length - 3, 3) == "jpg")
                            {
                                try
                                {
                                    string guid = Guid.NewGuid().ToString();
                                    webClient.DownloadFile(MatchList[j].Groups[1].Value, @"c:\temp\" + guid + ".jpg");

                                    FileInfo fi = new FileInfo(@"c:\temp\" + guid + ".jpg");
                                    this.image_size = fi.Length;

                                    using (Image img = Image.FromFile(@"c:\temp\" + guid + ".jpg"))
                                    {
                                        int minHeight = this.cov_min_height;
                                        if ((bool)((object[])options)[1])
                                            minHeight = 100;

                                        if (img.RawFormat.Equals(System.Drawing.Imaging.ImageFormat.Jpeg) && img.HorizontalResolution > 70 && img.Size.Height > minHeight && img.Size.Width > this.cov_min_width && this.image_size < 250000)
                                        {
                                            foundgg = true;
                                            image_name = guid;
                                            image_height = img.Height;
                                            image_width = img.Width;
                                            img.Dispose();
                                            if ((bool)((object[])options)[0])
                                            {
                                                Mediatly.savePoster(this, (bool)((object[])options)[1]);
                                            }
                                        }
                                        else
                                        {
                                            img.Dispose();
                                            File.Delete(@"c:\temp\" + guid.ToString() + ".jpg"); 
                                        }
                                    }
                                }
                                catch (Exception ex)
                                {
                                }
                            }

                            j++;
                        }
                    }
                }

                webClient.Dispose();
                Log.CountImgDec();
            }
        }
    }

    public void getPosterAsc(bool save = false, bool banner = false)
    {
        ThreadPool.QueueUserWorkItem(new WaitCallback(getPoster), new object[2] { save, banner });
    }
+3
source share
4 answers

I would make sure you use the thread pool to manage your threads. Since someone said that each thread consumes about 1 MB of memory, and this can cause problems depending on the system hardware.

- . , , . ( ), .

( , ), . " ". , , , , . , , , , .

:

  • BackgroundWorker.
  • ThreadPool.QueueUserWorkItem.
  • .
  • PLINQ.

TPL, ! , .

+2

perfmon, , , " ". , - () - .

Windows 2008 dynacache Microsoft, .

.net, , , , ( )

+1
0

, . , "" . , Out of Memory

ddlValue = webClientProxy.DownloadString((string)((object[])obj)[1]);

, - , . .

, , , , . , , , .

0

All Articles