System.OutOfMemoryExeption C#

Question

When I'm running my application and load a file(25MB) through it, everything runs just fine.

But when I try loading a file(160MB) I get a System.OutOfMemoryExeption. Although I have been able to load the larger file at some point in time.

Is there anyway to fix this? If so, any help would be much appreciated!

My Code that loads the files:

private void openFile (string fileName)
    {
        List<Structs.strValidData> _header1 = new List<Structs.strValidData>();
        List<Structs.strValidData> _header2 = new List<Structs.strValidData>();
        List<Structs.strValidData> _header3 = new List<Structs.strValidData>();
        List<Structs.strValidData> _header4 = new List<Structs.strValidData>();

        var textBoxArray = new[]
                {
                    textBoxResStart_Status1,
                    textBoxResStart_Status2,
                    textBoxResStart_Status3,
                    textBoxResStart_Status4,
                    textBoxResStart_Status5,
                    textBoxResStart_Status6,
                    textBoxResStart_Status7,
                    textBoxResStart_Status8,
                };

        var radioButtonArray = new[]
                {
                    radioButtonResStart_SelectStr1,
                    radioButtonResStart_SelectStr2,
                    radioButtonResStart_SelectStr3,
                    radioButtonResStart_SelectStr4,
                    radioButtonResStart_SelectStr5,
                    radioButtonResStart_SelectStr6,
                    radioButtonResStart_SelectStr7,
                    radioButtonResStart_SelectStr8,
                };

        readCSV read;
        read = new readCSV();

        strInfo = default(Structs.strInfo);
        strData = default(Structs.strData);
        strSetup = default(Structs.strSetup);
        strValidData = new List<Structs.strValidData>();
        readID = default(Structs.ReadID);

        try
        {
            strInfo = read.loadInfo(fileName);
            strData = read.loadData(fileName);
            strSetup = read.loadSetup(fileName);
            readID = read.loadID(fileName);

            strValidData = read.loadValidData(fileName);

            var Str1 = read.loadStr1(fileName);
            var Str235678 = read.loadStr235678(fileName);
            var Str4 = read.loadStr4(fileName);

            foreach (Structs.strValidData items in strValidData)
            {
                if (items.Str1_ValidData == true)
                {
                    Str1_headers.Add(items);
                }

                if (items.Str2_ValidData == true ||
                    items.Str3_ValidData == true ||
                    items.Str5_ValidData == true ||
                    items.Str6_ValidData == true ||
                    items.Str7_ValidData == true ||
                    items.Str8_ValidData == true)
                {
                    Str235678_headers.Add(items);
                }

                if (items.Str4_ValidData == true)
                {
                    Str4_headers.Add(items);
                }
            }

            Str1_data = combineData(Str1, Str1_headers);
            Str4_data = combineData(Str4, Str4_headers);
            var Str235678_CombinedData = combineData(Str235678, Str235678_headers);


            foreach (Structs.strValidData items in Str235678_CombinedData)
            {
                if (items.Str2_ValidData == true)
                {
                    Str2_data.Add(items);
                }
                if (items.Str3_ValidData == true)
                {
                    Str3_data.Add(items);
                }
                if (items.Str5_ValidData == true)
                {
                    Str5_data.Add(items);
                }
                if (items.Str6_ValidData == true)
                {
                    Str6_data.Add(items);
                }
                if (items.Str7_ValidData == true)
                {
                    Str7_data.Add(items);
                }
                if (items.Str8_ValidData == true)
                {
                    Str8_data.Add(items);
                }

            }

            strInfo = read.loadInfo(openDialog.FileName);
            strData = read.loadData(openDialog.FileName);
            strSetup = read.loadSetup(openDialog.FileName);
            readID = read.loadID(openDialog.FileName);

        }

        catch (Exception err)
        {
            MessageBox.Show(err.Message);
            error.logSystemError(err);
        }
    }

Here are the ReadCSV() code:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using FileHelpers;
using FileHelpers.Events;

namespace Reading_Files
{


public class readCSV
{

    public int strCnt = 0;
    private readCSVprogressForm _waitForm;

    public List<Structs.strDataImport> copyList(List<strData> copyFrom)
    {
        List<Structs.strDataImport> list = new List<Structs.strDataImport>();

        list.AddRange(copyFrom.Select(s => copyListContents(s)));

        return list;
    }

    public Structs.strDataImport copyListContents(strData copyFrom)
    {
        Structs.strDataImport data = new Structs.strDataImport();
        data.sCD_TimeCP2711 = copyFrom.sCD_TimeCP2711;
        data.sCD_TimeCX9020_1 = copyFrom.sCD_TimeCX9020_1;
        data.sCD_TimeCX9020_2 = copyFrom.sCD_TimeCX9020_2;
        data.rCD_CX9020_1_TimeDiff_DataLow = (Int32)(copyFrom.rCD_CX9020_1_TimeDiff_DataLow);
        data.rCD_CX9020_2_TimeDiff_DataLow = (Int32)(copyFrom.rCD_CX9020_2_TimeDiff_DataLow);
        data.iCD_NumUpper = copyFrom.iCD_NumUpper;
        data.iCD_NumUpper = copyFrom.iCD_NumUpper;
        data.iCD_NumLower = copyFrom.iCD_NumLower;
        data.iCD_NumLower = copyFrom.iCD_NumLower;

        data.bCD_1_Status = copyFrom.bCD_1_Status;
        data.bCD_1_Overrange = copyFrom.bCD_1_Overrange;
        data.iCD_1_Str_ID = copyFrom.iCD_1_Str_ID;
        data.rCD_1_Value = copyFrom.rCD_1_Value;

        data.bCD_2_Status = copyFrom.bCD_2_Status;
        data.bCD_2_Overrange = copyFrom.bCD_2_Overrange;
        data.iCD_2_Str_ID = copyFrom.iCD_2_Str_ID;
        data.rCD_2_Value = copyFrom.rCD_2_Value;

        data.bCD_3_Status = copyFrom.bCD_3_Status;
        data.bCD_3_Overrange = copyFrom.bCD_3_Overrange;
        data.iCD_3_Str_ID = copyFrom.iCD_3_Str_ID;
        data.iCD_3_RawData = copyFrom.iCD_3_RawData;
        data.rCD_3_Value = copyFrom.rCD_3_Value;

        data.bCD_4_Status = copyFrom.bCD_4_Status;
        data.bCD_4_Overrange = copyFrom.bCD_4_Overrange;
        data.iCD_4_Str_ID = copyFrom.iCD_4_Str_ID;
        data.iCD_4_RawData = copyFrom.iCD_4_RawData;
        data.rCD_4_Value = copyFrom.rCD_4_Value;

        data.bCD_5_Status = copyFrom.bCD_5_Status;
        data.bCD_5_Overrange = copyFrom.bCD_5_Overrange;
        data.iCD_5_Str_ID = copyFrom.iCD_5_Str_ID;
        data.iCD_5_RawData = copyFrom.iCD_5_RawData;
        data.rCD_5_Value = copyFrom.rCD_5_Value;

        data.bCD_6_Status = copyFrom.bCD_6_Status;
        data.bCD_6_Overrange = copyFrom.bCD_6_Overrange;
        data.iCD_6_Str_ID = copyFrom.iCD_6_Str_ID;
        data.iCD_6_RawData = copyFrom.iCD_6_RawData;
        data.rCD_6_Value = copyFrom.rCD_6_Value;

        data.bCD_7_Status = copyFrom.bCD_7_Status;
        data.bCD_7_Overrange = copyFrom.bCD_7_Overrange;
        data.iCD_7_Str_ID = copyFrom.iCD_7_Str_ID;
        data.iCD_7_RawData = copyFrom.iCD_7_RawData;
        data.rCD_7_Value = copyFrom.rCD_7_Value;

        data.bCD_8_Status = copyFrom.bCD_8_Status;
        data.bCD_8_Overrange = copyFrom.bCD_8_Overrange;
        data.iCD_8_Str_ID = copyFrom.iCD_8_Str_ID;
        data.iCD_8_RawData = copyFrom.iCD_8_RawData;
        data.rCD_8_Value = copyFrom.rCD_8_Value;

        data.bCD_9_Status = copyFrom.bCD_9_Status;
        data.bCD_9_Overrange = copyFrom.bCD_9_Overrange;
        data.iCD_9_Str_ID = copyFrom.iCD_9_Str_ID;
        data.iCD_9_RawData = copyFrom.iCD_9_RawData;
        data.rCD_9_Value = copyFrom.rCD_9_Value;

        data.bCD_10_Status = copyFrom.bCD_10_Status;
        data.bCD_10_Overrange = copyFrom.bCD_10_Overrange;
        data.iCD_10_Str_ID = copyFrom.iCD_10_Str_ID;
        data.iCD_10_RawData = copyFrom.iCD_10_RawData;
        data.rCD_10_Value = copyFrom.rCD_10_Value;

        data.bCD_11_Status = copyFrom.bCD_11_Status;
        data.bCD_11_Overrange = copyFrom.bCD_11_Overrange;
        data.iCD_11_Str_ID = copyFrom.iCD_11_Str_ID;
        data.iCD_11_RawData = copyFrom.iCD_11_RawData;
        data.rCD_11_Value = copyFrom.rCD_11_Value;

        data.bCD_12_Status = copyFrom.bCD_12_Status;
        data.bCD_12_Overrange = copyFrom.bCD_12_Overrange;
        data.iCD_12_Str_ID = copyFrom.iCD_12_Str_ID;
        data.iCD_12_RawData = copyFrom.iCD_12_RawData;
        data.rCD_12_Value = copyFrom.rCD_12_Value;

        data.bCD_13_Status = copyFrom.bCD_13_Status;
        data.bCD_13_Overrange = copyFrom.bCD_13_Overrange;
        data.iCD_13_Str_ID = copyFrom.iCD_13_Str_ID;
        data.iCD_13_RawData = copyFrom.iCD_13_RawData;
        data.rCD_13_Value = copyFrom.rCD_13_Value;

        data.bCD_14_Status = copyFrom.bCD_14_Status;
        data.bCD_14_Overrange = copyFrom.bCD_14_Overrange;
        data.iCD_14_Str_ID = copyFrom.iCD_14_Str_ID;
        data.iCD_14_RawData = copyFrom.iCD_14_RawData;
        data.rCD_14_Value = copyFrom.rCD_14_Value;

        data.bCD_15_Status = copyFrom.bCD_15_Status;
        data.bCD_15_Overrange = copyFrom.bCD_15_Overrange;
        data.iCD_15_Str_ID = copyFrom.iCD_15_Str_ID;
        data.iCD_15_RawData = copyFrom.iCD_15_RawData;
        data.rCD_15_Value = copyFrom.rCD_15_Value;

        data.bCD_16_Status = copyFrom.bCD_16_Status;
        data.bCD_16_Overrange = copyFrom.bCD_16_Overrange;
        data.iCD_16_Str_ID = copyFrom.iCD_16_Str_ID;
        data.iCD_16_RawData = copyFrom.iCD_16_RawData;
        data.rCD_16_Value = copyFrom.rCD_16_Value;

        data.bCD_17_Status = copyFrom.bCD_17_Status;
        data.bCD_17_Overrange = copyFrom.bCD_17_Overrange;
        data.iCD_17_Str_ID = copyFrom.iCD_17_Str_ID;
        data.iCD_17_RawData = copyFrom.iCD_17_RawData;
        data.rCD_17_Value = copyFrom.rCD_17_Value;

        data.bCD_18_Status = copyFrom.bCD_18_Status;
        data.bCD_18_Overrange = copyFrom.bCD_18_Overrange;
        data.iCD_18_Str_ID = copyFrom.iCD_18_Str_ID;
        data.iCD_18_RawData = copyFrom.iCD_18_RawData;
        data.rCD_18_Value = copyFrom.rCD_18_Value;

        data.bCD_19_Status = copyFrom.bCD_19_Status;
        data.bCD_19_Overrange = copyFrom.bCD_19_Overrange;
        data.iCD_19_Str_ID = copyFrom.iCD_19_Str_ID;
        data.rCD_19_Value = copyFrom.rCD_19_Value;

        data.bCD_20_Status = copyFrom.bCD_20_Status;
        data.bCD_20_Overrange = copyFrom.bCD_20_Overrange;
        data.iCD_20_Str_ID = copyFrom.iCD_20_Str_ID;
        data.rCD_20_Value = copyFrom.rCD_20_Value;

        return data;
    }

    public Structs.ReaStrID load_ID(string FileName)
    {
        var engine = new MultiRecordEngine(typeof(strInfo),
                            typeof(strData),
                            typeof(strSetup),
                            typeof(strID),
                            typeof(strData));

        engine.RecordSelector = new RecordTypeSelector(strIDSelector);
        var data = engine.ReadFile(FileName);

        Structs.ReaStrID structure = new Structs.ReaStrID();
        foreach (strID filteredData in data)
        {
            structure.steID[0] = filteredData._1_Ste_ID;
            structure.Status[0] = filteredData._1_Status;
            structure.steID[1] = filteredData._2_Ste_ID;
            structure.Status[1] = filteredData._2_Status;
            structure.steID[2] = filteredData._3_Ste_ID;
            structure.Status[2] = filteredData._3_Status;
            structure.steID[3] = filteredData._4_Ste_ID;
            structure.Status[3] = filteredData._4_Status;
            structure.steID[4] = filteredData._5_Ste_ID;
            structure.Status[4] = filteredData._5_Status;
        }

        return structure;
    }

    public Structs.strInfo loadInfo(string FileName)
    {
        var engine = new MultiRecordEngine(typeof(strInfo),
                            typeof(strData),
                            typeof(strSetup),
                            typeof(strID),
                            typeof(strData));

        engine.RecordSelector = new RecordTypeSelector(strInfoSelector);
        var data = engine.ReadFile(FileName);

        Structs.strInfo structure = new Structs.strInfo();
        foreach (strInfo filteredData in data)
        {
            structure.Date = filteredData.Date;
            structure.Description1 = filteredData.Description1;
            structure.Description2 = filteredData.Description2;
            structure.Description3 = filteredData.Description3;
        }

        return structure;
    }

    public Structs.strData loadData(string FileName)
    {
        var engine = new MultiRecordEngine(typeof(strInfo),
                            typeof(strData),
                            typeof(strSetup),
                            typeof(strID),
                            typeof(strData));

        engine.RecordSelector = new RecordTypeSelector(strDataSelector);
        var data = engine.ReadFile(FileName);

        Structs.strData structure = new Structs.strData();
        foreach (strData filteredData in data)
        {
            structure.iMDstr_var1_TypeID = filteredData.iMDstr_var1_TypeID;
            structure.rMDstr_var1_Lenght = filteredData.rMDstr_var1_Lenght;
            structure.iMDstr_var2_TypeID = filteredData.iMDstr_var2_TypeID;
            structure.rMDstr_var2_Lenght = filteredData.rMDstr_var2_Lenght;
            structure.iMDstr_var3_TypeID = filteredData.iMDstr_var3_TypeID;
            structure.rMDstr_var3_Lenght = filteredData.rMDstr_var3_Lenght;
            structure.iMDstr_var4_TypeID = filteredData.iMDstr_var4_TypeID;
            structure.rMDstr_var4_Lenght = filteredData.rMDstr_var4_Lenght;

        }

        return structure;
    }

    public Structs.strSetup loadSetup(string FileName)
    {
        var engine = new MultiRecordEngine(typeof(strInfo),
                            typeof(strID),
                            typeof(strData),
                            typeof(strSetup),
                            typeof(strData));

        engine.RecordSelector = new RecordTypeSelector(strSetupSelector);
        var data = engine.ReadFile(FileName);

        Structs.strSetup structure = new Structs.strSetup();

        foreach (strSetup filteredData in data)
        {
            structure.sSSstr_Sens = filteredData.sSSstr_Sens;
            structure.bSSstr_S1_A = filteredData.bSSstr_S1_A;
            structure.iSSstr_S1_B = filteredData.iSSstr_S1_B;
            structure.sSSstr_S1_C = filteredData.sSSstr_S1_C;
            structure.rSSstr_S1_D = filteredData.rSSstr_S1_D;
            structure.bSSstr_S2_A = filteredData.bSSstr_S2_A;
            structure.iSSstr_S2_B = filteredData.iSSstr_S2_B;
            structure.sSSstr_S2_C = filteredData.sSSstr_S2_C;
            structure.rSSstr_S2_D = filteredData.rSSstr_S2_D;
            structure.bSSstr_S3_A = filteredData.bSSstr_S3_A;
            structure.iSSstr_S3_B = filteredData.iSSstr_S3_B;
            structure.sSSstr_S3_C = filteredData.sSSstr_S3_C;
            structure.iSSstr_S3_D = filteredData.iSSstr_S3_D;
        }
        return structure;
    }

    public List<Structs.str1> load1(string FileName)
    {
        var engine = new MultiRecordEngine(typeof(strInfo),
                            typeof(strData),
                            typeof(strSetup),
                            typeof(strData),
                            typeof(strID),
                            typeof(strValidData),
                            typeof(strStartNum),
                            typeof(str1),
                            typeof(str4));

        engine.RecordSelector = new RecordTypeSelector(str1Selector);
        var data = engine.ReadFile(FileName);

        List<Structs.str1> list = new List<Structs.str1>();
        int i = 0;
        foreach (str1 data1 in data)
        {
            Structs.str1 structure = new Structs.str1();
            structure.rGL_1_L_Positive = data1.rGL_1_L_Positive;
            structure.rGL_1_L_Negative = data1.rGL_1_L_Negative;
            structure.rGL_1_R_Positive = data1.rGL_1_R_Positive;
            structure.rGL_1_R_Negative = data1.rGL_1_R_Negative;

            list.Add(structure);
            i++;
        }

        return list;
    }

    public List<Structs.str4> load4(string FileName)
    {
        var engine = new MultiRecordEngine(typeof(strInfo),
                            typeof(strData),
                            typeof(strSetup),
                            typeof(strData),
                            typeof(strValidData),
                            typeof(strStartNum),
                            typeof(str1),
                            typeof(str4));

        engine.RecordSelector = new RecordTypeSelector(str4Selector);
        var data = engine.ReadFile(FileName);

        List<Structs.str4> list = new List<Structs.str4>();
        int i = 0;
        foreach (str4 data4 in data)
        {
            Structs.str4 structure = new Structs.str4();

            structure.rGL_4_1 = data4.rGL_4_1;
            structure.rGL_4_2 = data4.rGL_4_2;
            structure.rGL_4_3 = data4.rGL_4_3;
            structure.rGL_4_4 = data4.rGL_4_4;
            structure.rGL_4_5 = data4.rGL_4_5;
            structure.rGL_4_6 = data4.rGL_4_6;
            structure.rGL_4_7 = data4.rGL_4_7;
            structure.rGL_4_8 = data4.rGL_4_8;

            list.Add(structure);
            i++;
        }

        return list;
    }

    public List<Structs.strValidData> loadValidData(string FileName)
    {
        var engine = new MultiRecordEngine(typeof(strInfo),
                            typeof(strData),
                            typeof(strSetup),
                            typeof(strID),
                            typeof(strData),
                            typeof(strValidData));

        engine.RecordSelector = new RecordTypeSelector(strValidDataSelector);
        var data = engine.ReadFile(FileName);

        List<Structs.strValidData> list = new List<Structs.strValidData>();
        int i = 0;
        foreach (strValidData strValidData in data)
        {
            Structs.strValidData structure = new Structs.strValidData();
            structure._name = String.Format("strItem {0}", i + 1);
            structure._index = i;
            structure.str1_ValidData = strValidData.str1_ValidData;
            structure.str2_ValidData = strValidData.str2_ValidData;
            structure.str3_ValidData = strValidData.str3_ValidData;
            structure.str4_ValidData = strValidData.str4_ValidData;
            structure.str5_ValidData = strValidData.str5_ValidData;
            structure.str6_ValidData = strValidData.str6_ValidData;
            structure.str7_ValidData = strValidData.str7_ValidData;
            structure.str8_ValidData = strValidData.str8_ValidData;
            structure.str9_ValidData = strValidData.str9_ValidData;
            list.Add(structure);
            i++;
        }

        return list;
    }

    public List<List<Structs.strDataImport>> loadstrDataAsync(string FileName)
    {
        var engine_Data = new FileHelperAsyncEngine<strData>();
        engine_Data.BeforeReadRecord += BeforeEventAsync;
        engine_Data.AfterReadRecord += AfterEventAsync;

        engine_Data.Progress += ReadProgress;

        List<strData> list = new List<strData>();
        List<List<Structs.strDataImport>> list2D = new List<List<Structs.strDataImport>>();

        using (engine_Data.BeginReadFile(FileName))
        {
            var prevRowNo = 0;
            var j = 0;
            strCnt = 0;
            foreach (strData filteredData in engine_Data)
            {
                if (prevRowNo > filteredData.RowNo)
                {
                    list2D.Add(copyList(list));
                    list.Clear();
                }
                prevRowNo = filteredData.RowNo;
                list.Add(filteredData);
            }
            list2D.Add(copyList(list));

        }

        return list2D;

    }

    private Type strIDSelector(MultiRecordEngine engine, string recordLine)
    {
        if (recordLine.Length == 0)
            return null;
        if (recordLine.Contains("** #_DATA .STATUS .STRID **"))
            return typeof(strID);
        else
        {
            return null;
        }

    }

    private Type InfoSelector(MultiRecordEngine engine, string recordLine)
    {
        if (recordLine.Length == 0)
            return null;
        if (recordLine.Contains("** #_FILE **"))
            return typeof(strInfo);
        else
        {
            return null;
        }

    }

    private Type strDataSelector(MultiRecordEngine engine, string recordLine)
    {
        if (recordLine.Length == 0)
            return null;

        if (recordLine.Contains("** #_DATA **"))
            return typeof(strData);
        else
        {
            return null;
        }

    }

    private Type strSetupSelector(MultiRecordEngine engine, string recordLine)
    {
        if (recordLine.Length == 0)
            return null;

        if (recordLine.Contains("** #_SETUP **"))
            return typeof(strSetup);
        else
        {
            return null;
        }

    }

    private Type strValidDataSelector(MultiRecordEngine engine, string recordLine)
    {
        if (recordLine.Length == 0)
            return null;

        if (recordLine.Contains("** #_VALID_DATA **"))
            return typeof(strValidData);
        else
        {
            return null;
        }

    }

    private Type StartNumSelector(MultiRecordEngine engine, string recordLine)
    {
        if (recordLine.Length == 0)
            return null;

        if (recordLine.Contains("** #_START_NUMBER **"))
            return typeof(strStartNum);
        else
        {
            return null;
        }

    }

    private Type str1Selector(MultiRecordEngine engine, string recordLine)
    {
        if (recordLine.Length == 0)
            return null;

        if (recordLine.Contains("** #_1 **"))
            return typeof(str1);
        else
        {
            return null;
        }

    }

    private Type str4Selector(MultiRecordEngine engine, string recordLine)
    {
        if (recordLine.Length == 0)
            return null;

        if (recordLine.Contains("** #_4 **"))
            return typeof(str4);
        else
        {
            return null;
        }

    }

    private void BeforeEventAsync(EngineBase engine, BeforeReadEventArgs<strData> e)
    {
        if (e.RecordLine != "")
        {
            if (Char.IsDigit(e.RecordLine, 0))
            {

            }
            else
            {
                e.SkipThisRecord = true;
            }
        }

        if (e.RecordLine.Contains("** #_VALID_DATA **;"))
        {
            e.SkipThisRecord = true;
        }




    }

    private void AfterEventAsync(EngineBase engine, AfterReadEventArgs<strData> e)
    {

        if (e.RecordLine.Contains("** #_VALID_DATA **;"))
        {
            e.SkipThisRecord = true;
        }


    }

    private void ReadProgress(object sender, ProgressEventArgs e)
    {
        ShowWaitForm("Opening file." + "\n" + "\n" + "Please wait...", "Open File");
        _waitForm.progressBar1.Value = Convert.ToInt16(e.Percent);
    }

    public void ShowWaitForm(string message, string caption)
    {
        if (_waitForm != null && !_waitForm.IsDisposed)
        {
            return;
        }

        _waitForm = new readCSVprogressForm();
        _waitForm.ShowMessage(message); 
        _waitForm.Text = caption;
        _waitForm.TopMost = true;
        _waitForm.Show();
        _waitForm.Refresh();


        System.Threading.Thread.Sleep(700);
        Application.Idle += OnLoaded;

    }

    private void OnLoaded(object sender, EventArgs e)
    {
        Application.Idle -= OnLoaded;
        _waitForm.Close();
    }


}

}

What kind of a file are you trying to lad onto memory? What kind of code are you using? Please show us what you have done so far? — Hasan Emrah Süngü
– Hasan Emrah Süngü, Commented Aug 30, 2018 at 9:12
That's pretty normal, the longer a .NET process has been running the more its address space gets fragmented. Running out of nice big holes in which you can shoehorn the data from that big file. You'll either need to write smarter code that processes the data in the file one record or line at a time. Or use Project > Properties > Build tab, untick "Prefer 32-bit". You don't prefer it. Using a memory profiler to keep yourself honest isn't going to hurt you btw. — Hans Passant
– Hans Passant, Commented Aug 30, 2018 at 9:17
OutOfMemoryException often occurs if you read a whole file into a string, e.g. by loading everything into a XDocument instead of using XMLReader. string needs to allocate a coherent memory segment without holes. So even if you got 4 Gb of free RAM, you might run into OutOfMemoryException when loading 160 Mb because the available memory is fragmented. — Georg Patscheider
– Georg Patscheider, Commented Aug 30, 2018 at 9:34
I'm really concerned by the shear number of methods that all apparently need to be passed the file name. How many times are you actually opening/reading/closing the file over the course of this block of code? — Damien_The_Unbeliever
– Damien_The_Unbeliever, Commented Aug 30, 2018 at 9:48

Jon Skeet · Accepted Answer · 2018-08-31 10:15:58Z

1

Given the comments, it sounds like the problem is really that you're using structs extensively. These should almost certainly be classes for idiomatic C#. See the design guidelines for more details of how to pick between these.

At the moment, you're loading all the values in a big List<T>. Internally, that has an array - so it's going to be an array of your struct type. That means all the values are required to be in a single contiguous chunk of memory - and it sounds like that chunk can't be allocated.

If you change your data types to classes, then a contiguous chunk of memory will still be required - but only enough to store references to the objects you create. You'll end up using slightly more data overall (due to per-object overhead and the references to those objects) but you won't have nearly as strict a requirement on allocating a single big chunk of memory.

That's only one reason to use classes here - the reason of "this just isn't a normal use of structs" is a far bigger one, IMO.

As an aside, I'd also very strongly recommend that you start following .NET naming conventions, particularly around the use of capitalization and avoiding underscores to separate words in names. (There are other suggestions for improving the code in the question too, and I'd advise reading them all carefully.)

answered Aug 31, 2018 at 10:15

Jon Skeet

1.5m893 gold badges9.3k silver badges9.3k bronze badges

Sign up to request clarification or add additional context in comments.

1 Comment

Red-X Over a year ago

Thanks for all your help! :) I'm still learning, so I really appreciate your advice and patience.

Collectives™ on Stack Overflow

System.OutOfMemoryExeption C#

1 Answer 1

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related