详解C#如何利用爬虫技术实现快捷租房

private readonly string douBanUrl = "https://www.douban.com/group/search?cat=1019&sort=time";

        bool isStop = false;

        public FormCrawler()
        {
            InitializeComponent();
        }

        private void btn_start_Click(object sender, EventArgs e)
        {
            if (string.IsNullOrWhiteSpace(txt_city.Text))
            {
                MessageBox.Show("请输入城市！");
                return;
            }
            if (string.IsNullOrWhiteSpace(txt_keys.Text))
            {
                MessageBox.Show("请输入关键字！");
                return;
            }

            //初始化控件
            btn_start.Enabled = false;
            btn_stop.Enabled = true;
            listView1.Items.Clear();
            progressBar1.Value = 0;

            List<string> groups = GetGroupUrls();
            progressBar1.Maximum = groups.Count;
            Task.Run(() =>
            {
                GetHouseInfo(groups);

                this.BeginInvoke(new Action(() =>
                {
                    btn_start.Enabled = true;
                    btn_stop.Enabled = false;
                }));
            });


        }

        private void btn_stop_Click(object sender, EventArgs e)
        {
            isStop = true;
            btn_start.Enabled = true;
            btn_stop.Enabled = false;
            progressBar1.Value = progressBar1.Maximum;
        }

     
        private void listView1_MouseDoubleClick(object sender, MouseEventArgs e)
        {
            ListView.SelectedListViewItemCollection selectItem = listView1.SelectedItems;
            if (selectItem.Count > 0)
            {
                Process.Start(selectItem[0].SubItems[1].Text);
            }
        }

/// <summary>
        /// 获取前100个讨论组
        /// </summary>
        /// <returns></returns>
        private List<string> GetGroupUrls()
        {
            List<string> groupUrls = new List<string>();

            int pageSize = 20;//豆瓣每页显示20个讨论组
            string groupUrl = string.Empty;

            for (int groupNum = 0; groupNum <= 80; groupNum += pageSize)
            {
                if (isStop) break;

                groupUrl = string.Format(douBanUrl + "&q={0}&start={1}", HttpUtility.UrlEncode(txt_city.Text + "租房", Encoding.UTF8), groupNum);

                string text = HttpUtil.HttpGet(groupUrl);

                if (!string.IsNullOrWhiteSpace(text))
                {
                    Regex reg = new Regex(@"(?is)(?<=<div\sclass=""pic""[^>]*?>).*?(?=</div>)");
                    MatchCollection matchs = reg.Matches(text);
                    for (int i = 0; i < matchs.Count; i++)
                    {
                        if (matchs[i].Success)
                        {
                            Regex regHref = new Regex("(?<=href\\s*=\\s*\")\\S+(?<!\")");
                            string href = regHref.Match(matchs[i].Value).Value;
                            groupUrls.Add(href);
                        }
                    }
                }
            }

            return groupUrls;
        }

        /// <summary>
        /// 获取讨论组内符合条件的房源
        /// </summary>
        /// <param name="urls"></param>
        private void GetHouseInfo(List<string> urls)
        {
            for (int u = 0; u < urls.Count; u++)
            {
                if (isStop) break;
                this.BeginInvoke(new Action(() =>
                {
                    progressBar1.Value++;
                }));

                string text = HttpUtil.HttpGet(urls[u] + "/discussion?start=");
                Regex regex = new Regex(@"(?is)(?<=<tr\sclass=""""[^>]*?>).*?(?=</tr>)");
                MatchCollection matchs = regex.Matches(text);
                for (int i = 0; i < matchs.Count; i++)
                {
                    if (matchs[i].Success)
                    {
                        #region 匹配时间
                        Regex regDate = new Regex(@"(?is)(?<=<td\snowrap=""nowrap""\sclass=""time""[^>]*?>).*?(?=</td>)");
                        Match matchDate = regDate.Match(matchs[i].Value);
                        if (!matchDate.Success)
                        {
                            continue;
                        }
                        string time = matchDate.Value;
                        DateTime dtTemp = new DateTime();
                        if (!DateTime.TryParse(time, out dtTemp))
                        {
                            time = time.Insert(0, DateTime.Now.Year + "-");
                            dtTemp = Convert.ToDateTime(time);
                        }
                        if (dtTemp < dateTimePicker1.Value.Date)
                        {
                            continue;
                        }
                        #endregion

                        #region 获取标题

                        Regex regTitle = new Regex("(?<=title\\s*=\\s*\")\\s*\\S*(?<!\")");
                        Match matchTitle = regTitle.Match(matchs[i].Value);
                        if (matchTitle.Success)
                        {
                            string title = matchTitle.Value;
                            #region 匹配标题
                            bool isContain = false;
                            string[] keys = txt_keys.Text.Split('*');
                            foreach (string key in keys)
                            {
                                if (matchTitle.Value.Contains(key))
                                {
                                    isContain = true;
                                }
                            }
                            #endregion
                            if (isContain)
                            {
                                //获取链接
                                Regex regHref = new Regex("(?<=href\\s*=\\s*\")\\s*\\S*(?<!\")");
                                string href = regHref.Match(matchs[i].Value).Value;

                                AddUI(title, href, time);
                            }
                        }
                        #endregion
                    }

                }
            }
        }


        private void AddUI(string title, string href, string time)
        {
            if (this.InvokeRequired)
            {
                this.BeginInvoke(new Action(() => AddUI(title, href, time)));
            }
            else
            {
                ListViewItem lvitem = new ListViewItem(new string[3] { title, href, time });
                listView1.Items.Add(lvitem);
            }
        }