Linux join命令快速实现从大文件中匹配内容

(
  echo "No,姓名,auid,地址"
  seq 1 400000 | awk 'BEGIN{OFS=","} {
      printf "%d,user_name_%09d,auid_%09d,地球%d\n", $1, $1, $1, $1
  }'
) > bigfile.csv

# 对文件进行乱序处理
grep -v 姓名 bigfile.csv | shuf > shuffled.csv
# 为csv文件添加表头
sed -i '1i No,姓名,auid,地址' shuffled.csv

tail -n 1000 shuffled.csv | awk -F',' '{print $3}' > auid_list.txt

grep $(shuf -n 1 auid_list.txt) shuffled.csv | awk -F',' '{
    OFS = ","
    $1 = "@" $1 "@"
    $2 = "@" $2 "@"
    $4 = "@" $4 "@"
    print
}' | tee -a shuffled.csv
⇩⇩⇩　⇩⇩⇩
@295178@,@user_name_000295178@,auid_000295178,@地球295178@

apluser@FengYeHong-HP:0724$ grep -a auid_000295178 shuffled.csv
295178,user_name_000295178,auid_000295178,地球295178
@295178@,@user_name_000295178@,auid_000295178,@地球295178@

apluser@FengYeHong-HP:0724$ sort -t, -k3,3 shuffled.csv > all_info_sort.csv
apluser@FengYeHong-HP:0724$ tail all_info_sort.csv
399991,user_name_000399991,auid_000399991,地球399991
399992,user_name_000399992,auid_000399992,地球399992
399993,user_name_000399993,auid_000399993,地球399993
399994,user_name_000399994,auid_000399994,地球399994
399995,user_name_000399995,auid_000399995,地球399995
399996,user_name_000399996,auid_000399996,地球399996
399997,user_name_000399997,auid_000399997,地球399997
399998,user_name_000399998,auid_000399998,地球399998
399999,user_name_000399999,auid_000399999,地球399999
400000,user_name_000400000,auid_000400000,地球400000

apluser@FengYeHong-HP:0724$ sort auid_list.txt > auid_list_sort.csv
apluser@FengYeHong-HP:0724$ tail auid_list_sort.csv
auid_000395846
auid_000395932
auid_000395948
auid_000396121
auid_000396465
auid_000396566
auid_000397508
auid_000398221
auid_000398744
auid_000398933

apluser@FengYeHong-HP:0724$ wc -l auid_list_sort.csv
1000 auid_list_sort.csv
apluser@FengYeHong-HP:0724$ join -t, -1 1 -2 3 auid_list_sort.csv all_info_sort.csv > result.csv
apluser@FengYeHong-HP:0724$ wc -l result.csv
1001 result.csv
apluser@FengYeHong-HP:0724$ grep -a auid_000295178 result.csv
auid_000295178,295178,user_name_000295178,地球295178
auid_000295178,@295178@,@user_name_000295178@,@地球295178@

apluser@FengYeHong-HP:0724$ nkf --guess auid_list_sort.csv
ASCII (LF)
apluser@FengYeHong-HP:0724$ awk '{printf "%s\r\n", $0}' auid_list_sort.csv > auid_list_CRLF_sort.csv
apluser@FengYeHong-HP:0724$
apluser@FengYeHong-HP:0724$ nkf --guess auid_list_CRLF_sort.csv
ASCII (CRLF)
apluser@FengYeHong-HP:0724$ nkf --guess all_info_sort.csv
UTF-8 (LF)
apluser@FengYeHong-HP:0724$ # 👇👇👇join之后, 无任何输出👇👇👇
apluser@FengYeHong-HP:0724$ join -t, -1 1 -2 3 auid_list_CRLF_sort.csv all_info_sort.csv
apluser@FengYeHong-HP:0724$ # 👆👆👆join之后, 无任何输出👆👆👆

apluser@FengYeHong-HP:0724$ head result.csv
auid_000000333,333,user_name_000000333,地球333
auid_000000835,835,user_name_000000835,地球835
auid_000000922,922,user_name_000000922,地球922
auid_000001206,1206,user_name_000001206,地球1206
auid_000001436,1436,user_name_000001436,地球1436
auid_000001853,1853,user_name_000001853,地球1853
auid_000001925,1925,user_name_000001925,地球1925
auid_000002146,2146,user_name_000002146,地球2146
auid_000002195,2195,user_name_000002195,地球2195
auid_000002798,2798,user_name_000002798,地球2798

awk -F',' '{
	col1 = $1;
	for(i=2; i<=3; i++) {
		printf "%s,", $i
	}
	printf "%s", col1
	for(i=4; i<=NF; i++) {
		printf ",%s", $i
	}
	printf "\n"
}' result.csv > result_handle.csv

apluser@FengYeHong-HP:0724$ head result_handle.csv
333,user_name_000000333,auid_000000333,地球333
835,user_name_000000835,auid_000000835,地球835
922,user_name_000000922,auid_000000922,地球922
1206,user_name_000001206,auid_000001206,地球1206
1436,user_name_000001436,auid_000001436,地球1436
1853,user_name_000001853,auid_000001853,地球1853
1925,user_name_000001925,auid_000001925,地球1925
2146,user_name_000002146,auid_000002146,地球2146
2195,user_name_000002195,auid_000002195,地球2195
2798,user_name_000002798,auid_000002798,地球2798

awk -F, 'NR==FNR{auid_list[$1]; next} $3 in auid_list' auid_list.txt shuffled.csv > result_handle_awk.csv

apluser@FengYeHong-HP:0724$ diff <(sort result_handle_awk.csv) <(sort result_handle.csv)
apluser@FengYeHong-HP:0724$