C#实现更快读写超级大文件的方法详解

using System;
using System.IO;
using System.Threading.Tasks;
using System.Data.SqlClient;
using System.Collections.Concurrent;

class Program
{
    static void Main()
    {
        string path = "yourFile.tsv";
        ConcurrentQueue<string[]> queue = new ConcurrentQueue<string[]>();

        var producer = Task.Run(() =>
        {
            using (var reader = new StreamReader(path))
            {
                string line;
                while ((line = reader.ReadLine()) != null)
                {
                    var fields = line.Split('\t');
                    queue.Enqueue(fields);
                }
            }
        });

        var consumers = new Task[4];
        for (int i = 0; i < consumers.Length; i++)
        {
            consumers[i] = Task.Run(() =>
            {
                string connString = "Your connection string";
                using (var conn = new SqlConnection(connString))
                {
                    conn.Open();
                    while (true)
                    {
                        if (queue.TryDequeue(out string[] fields))
                        {
                            using (var cmd = new SqlCommand("Your insert command", conn))
                            {
                                // Update parameters in cmd using fields
                                cmd.ExecuteNonQuery();
                            }
                        }
                        else if (producer.IsCompleted)
                        {
                            break;
                        }
                    }
                }
            });
        }

        Task.WaitAll(consumers);
    }
}

[
	{
	"start":0,
	"end":10000
	},
	{
	"start":10001,
	"end":20000
	},
	{
	"start":20001,
	"end":30000
	},
	{
	"start":30000,
	"end":40000
	}
]

var data = new object []{
   
	{
	"start":0,
	"end":10000
	},
	{
	"start":10001,
	"end":20000
	},
	{
	"start":20001,
	"end":30000
	},
	{
	"start":30000,
	"end":40000
	}
}

// 处理偏移值的方法

// 提供多个线程任务去并发执行读