开源大数据去重复1E1000W数据只需30S左右 极小内存占用

Imports System.IO
Module Module1
    Sub Main()
        Console.Title = "SuperDeduplication V1.0 By:MythHack"
        Try
            Dim dir As String = AppDomain.CurrentDomain.BaseDirectory
            Dim proc_self As Process = Process.GetCurrentProcess()
            Console.Write("请输入文件名(不要加路径): ")
            Dim filename As String = Console.ReadLine
            Dim file1 As StreamReader = New StreamReader(dir & filename)
            Dim set1 As New HashSet(Of String)
            While file1.Peek > -1
                set1.Add(file1.ReadLine)
            End While
            file1.Close()
            Dim file2 As StreamWriter = New StreamWriter(dir & String.Format("done{0}.txt", DateTime.Now.Year & DateTime.Now.Month & DateTime.Now.Day & DateTime.Now.Hour & DateTime.Now.Minute & DateTime.Now.Second))
            Dim txtw As TextWriter = TextWriter.Synchronized(file2)
            Parallel.ForEach(set1, Sub(tm1)
                                       txtw.WriteLine(tm1)
                                   End Sub)
            file2.Close()
            Console.WriteLine("处理完成")
            Console.WriteLine(String.Format("处理时间:{0}", (DateTime.Now - proc_self.StartTime).ToString("T")))
            Console.ReadLine()
        Catch ex As Exception
            Console.WriteLine(String.Format("错误:{0}", ex.Message))
        End Try
    End Sub
End Module