Java获取网络文件并插入数据库的代码

 更新时间:2010年06月11日 00:51:52   作者:  
抓取各大网站的数据插入数据库,这样就不用为没有数据而烦恼了
获取百度的歌曲名,歌手和链接!!
复制代码 代码如下:

package webTools;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import dbTools.DBTools;
public class IOTOWeb {
public String getHtmlContent(String htmlURL) {
URL url = null;
String rowContent = "";
StringBuffer htmlContent = new StringBuffer();
try {
url = new URL(htmlURL);
BufferedReader in = new BufferedReader(new InputStreamReader(url
.openStream(), "gb2312"));
while ((rowContent = in.readLine()) != null) {
htmlContent.append(rowContent);
}
in.close();
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return htmlContent.toString();
}
public List getLink(String htmlContent) {
ArrayList listLink = new ArrayList();
String regex = "<td[^>]*>[\\(]*<a[^>]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.*?)[\\)]*[\\s]*</td>";
Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
Matcher matcher = pattern.matcher(htmlContent);
while (matcher.find()) {
listLink.add(matcher.group());
}
return listLink;
}
public List<String> getHref(String htmlContent) {
String regex;
List listtHref = new ArrayList();
regex = "href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))\"";
Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
Matcher ma = pa.matcher(htmlContent);
while (ma.find()) {
listtHref.add(ma.group().replaceFirst("href=\"", "").replace("\"",
""));
}
return listtHref;
}
public List<String> getPerson(String htmlContent) {
String regex;
List list = new ArrayList();
regex = "\\(<a[^>]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.*?)\\)";
Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
Matcher ma = pa.matcher(htmlContent);
while (ma.find()) {
list.add(ma.group().replaceFirst("href=\"", "").replace("\"", ""));
}
return list;
}
public List<String> getSongName(String htmlContent) {
String regex;
List listPerson = new ArrayList();
regex = "<a[^>]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.*?)</a>\\s";
Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
Matcher ma = pa.matcher(htmlContent);
while (ma.find()) {
listPerson.add(ma.group());
}
return listPerson;
}
public String getMainContent(String htmlContent) {
String regex = "<table width=\"100%\" align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"list\">(.*?)</table>";
StringBuffer mainContent = new StringBuffer();
Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
Matcher matcher = pattern.matcher(htmlContent);
while (matcher.find()) {
mainContent.append(matcher.group());
}
return mainContent.toString();
}
public String outTag(final String s) {
return s.replaceAll("<.*?>", "");
}
DBTools dbTools = new DBTools();
public void getFromBaiduMap3(String htmlURL) throws Throwable {
HashMap htmlContentMap = new HashMap();
String htmlContent = getHtmlContent(htmlURL);
String mainContent = getMainContent(htmlContent);
List listLink = getLink(mainContent);
for (int j = 0; j < listLink.size(); j++) {
String tdTag = listLink.get(j).toString();
List songNameList = getSongName(tdTag);
String songName = outTag(songNameList.get(0).toString());
List personList = getPerson(tdTag);
String songPerson = "";
if (personList.size() != 0) {
for (int n = 0; n < personList.size(); n++) {
// System.out.println(personList.get(n).toString());
songPerson = outTag(personList.get(n).toString());
}
} else {
songPerson = "无";
}
// System.out.print(songNameList.get(0).toString());
List hrefList = getHref(songNameList.get(0).toString());
String songHref = hrefList.get(0).toString();
System.out.println();
String sql = "insert into song(songName,songPerson,songHref) values(?,?,?)";
ArrayList list_values = new ArrayList();
list_values.add(songName);
list_values.add(songPerson);
list_values.add(songHref);
dbTools.update(sql, list_values);
}
}
}

DBTools数据库链接类:
复制代码 代码如下:

package dbTools;
import java.util.ArrayList;
import java.sql.*;
public class DBTools {
private PreparedStatement preparedStatement;
private ResultSet resultSet;
private Connection connection;
public DBTools() {
try {
Class.forName("com.mysql.jdbc.Driver");
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
connection = DriverManager.getConnection(
"jdbc:mysql://localhost:3306/TestURL", "root", "zhuyi");
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public ArrayList query(String sql, ArrayList list_values) throws Throwable {
ArrayList listRows = new ArrayList();
preparedStatement = connection.prepareStatement(sql);
for (int i = 0; i < list_values.size(); i++) {
preparedStatement.setObject(i + 1, list_values.get(i));
}
resultSet = preparedStatement.executeQuery();
while (resultSet.next()) {
String[] rowinfo = new String[resultSet.getMetaData()
.getColumnCount()];
for (int i = 0; i < rowinfo.length; i++) {
rowinfo[i] = resultSet.getString(i + 1);
}
listRows.add(rowinfo);
}
return listRows;
}
public void update(String sql, ArrayList list_values) throws Throwable {
preparedStatement = connection.prepareStatement(sql);
for (int i = 0; i < list_values.size(); i++) {
preparedStatement.setObject(i + 1, list_values.get(i));
}
preparedStatement.executeUpdate();
preparedStatement.close();
}
}

Servlet调用:
复制代码 代码如下:

package controller;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import webTools.IOTOWeb;
public class TestURL extends HttpServlet {
/**
* Constructor of the object.
*/
public TestURL() {
super();
}
/**
* Destruction of the servlet. <br>
*/
public void destroy() {
super.destroy(); // Just puts "destroy" string in log
// Put your code here
}
/**
* The doGet method of the servlet. <br>
*
* This method is called when a form has its tag value method equals to get.
*
* @param request
* the request send by the client to the server
* @param response
* the response send by the server to the client
* @throws ServletException
* if an error occurred
* @throws IOException
* if an error occurred
*/
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
IOTOWeb iotoWeb = new IOTOWeb();
iotoWeb.getFromBaiduMap3("http://list.mp3.baidu.com/topso/mp3topsong.html?id=1?top2");
} catch (Throwable e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
* The doPost method of the servlet. <br>
*
* This method is called when a form has its tag value method equals to
* post.
*
* @param request
* the request send by the client to the server
* @param response
* the response send by the server to the client
* @throws ServletException
* if an error occurred
* @throws IOException
* if an error occurred
*/
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
response.setContentType("text/html");
PrintWriter out = response.getWriter();
out
.println("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">");
out.println("<HTML>");
out.println(" <HEAD><TITLE>A Servlet</TITLE></HEAD>");
out.println(" <BODY>");
out.print(" This is ");
out.print(this.getClass());
out.println(", using the POST method");
out.println(" </BODY>");
out.println("</HTML>");
out.flush();
out.close();
}
/**
* Initialization of the servlet. <br>
*
* @throws ServletException
* if an error occurs
*/
public void init() throws ServletException {
// Put your code here
}
}

获取金书网的图书名:
复制代码 代码如下:

package webTools;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import dbTools.DBTools;
public class GetBook {
public String getHtmlContent(String htmlURL) throws Throwable {
URL url = null;
String rowContent = "";
StringBuffer htmlContent = new StringBuffer();
url = new URL(htmlURL);
BufferedReader in = new BufferedReader(new InputStreamReader(url
.openStream(), "gb2312"));
while ((rowContent = in.readLine()) != null) {
htmlContent.append(rowContent);
}
in.close();
return htmlContent.toString();
}
public String getBookName(String htmlContent) {
String bookName = "";
String regex = "<span class=\"style15\">[^>]*</span>";
Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
Matcher matcher = pattern.matcher(htmlContent);
if (matcher.find()) {
bookName = matcher.group();
}
return bookName;
}
public String outTag(final String s) {
return s.replaceAll("<.*?>", "");
}
DBTools dbtools = new DBTools();
public void getFromJINSHU(String htmlURL) throws Throwable {
String htmlContent = getHtmlContent(htmlURL);
String bookName = outTag(getBookName(htmlContent));
if (bookName != null && !"".equals(bookName)) {
System.out.println(bookName);
String sql = "insert into bookinfo(bookName) values(?)";
ArrayList list_values = new ArrayList();
list_values.add(bookName);
dbtools.update(sql, list_values);
}
}
}

调用Servlet:
复制代码 代码如下:

package controller;
import java.io.IOException;
import java.io.PrintWriter;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import webTools.GetBook;
public class TestBook extends HttpServlet {
/**
* Constructor of the object.
*/
public TestBook() {
super();
}
/**
* Destruction of the servlet. <br>
*/
public void destroy() {
super.destroy(); // Just puts "destroy" string in log
// Put your code here
}
/**
* The doGet method of the servlet. <br>
*
* This method is called when a form has its tag value method equals to get.
*
* @param request
* the request send by the client to the server
* @param response
* the response send by the server to the client
* @throws ServletException
* if an error occurred
* @throws IOException
* if an error occurred
*/
int i = 1;
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
GetBook bookinfo = new GetBook();
for (; i < 10000; i++) {
String bookURL = "http://www.golden-book.com/booksinfo/12/" + i
+ ".html";
try {
bookinfo.getFromJINSHU(bookURL);
} catch (Throwable e) {
i++;
doPost(request, response);
}
}
}
/**
* The doPost method of the servlet. <br>
*
* This method is called when a form has its tag value method equals to
* post.
*
* @param request
* the request send by the client to the server
* @param response
* the response send by the server to the client
* @throws ServletException
* if an error occurred
* @throws IOException
* if an error occurred
*/
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
GetBook bookinfo = new GetBook();
for (; i < 10000; i++) {
String bookURL = "http://www.golden-book.com/booksinfo/12/" + i
+ ".html";
try {
bookinfo.getFromJINSHU(bookURL);
} catch (Throwable e) {
i++;
doGet(request, response);
}
}
}
/**
* Initialization of the servlet. <br>
*
* @throws ServletException
* if an error occurs
*/
public void init() throws ServletException {
// Put your code here
}
}

每种功能的实现方法有很多,希望各位可以交流不同的思想和方法。可以加QQ412546724。呵呵

相关文章

  • java中forward转发的使用

    java中forward转发的使用

    在Java中,forward转发是一种非常常见且重要的操作,我们将深入探讨forward的概念和用法,并给出一些代码示例来帮助读者更好地理解,感兴趣的可以了解下
    2023-11-11
  • Java实现合并多个PDF的示例代码

    Java实现合并多个PDF的示例代码

    这篇文章主要介绍了通过Java实现合并多个PDF,并将合并后的新PDF存储到文件夹下,文中的示例代码简洁易懂,感兴趣的可以跟随小编一起试一试
    2022-01-01
  • 浅谈java中null是什么,以及使用中要注意的事项

    浅谈java中null是什么,以及使用中要注意的事项

    下面小编就为大家带来一篇浅谈java中null是什么,以及使用中要注意的事项。小编觉得挺不错的,现在就分享给大家,也给大家做个参考。一起跟随小编过来看看吧
    2017-09-09
  • SpringBoot定义过滤器、监听器、拦截器的方法

    SpringBoot定义过滤器、监听器、拦截器的方法

    本篇文章主要介绍了SpringBoot定义过滤器、监听器、拦截器的方法,具有一定的参考价值,有兴趣的可以了解一下。
    2017-04-04
  • Java Class 加密工具 ClassFinal详解

    Java Class 加密工具 ClassFinal详解

    ClassFinal 是一款 java class 文件安全加密工具,支持直接加密jar包或war包,无需修改任何项目代码,兼容spring-framework;可避免源码泄漏或字节码被反编译,这篇文章主要介绍了Java Class 加密工具 ClassFinal,需要的朋友可以参考下
    2023-03-03
  • MybatisPlus多表连接查询的问题及解决方案

    MybatisPlus多表连接查询的问题及解决方案

    MybatisPlus官方并没有提供多表连接查询的通用解决方案,然而连接查询是相当普遍的需求,今天通过本文给大家介绍下MybatisPlus多表连接查询的问题及解决方案,感兴趣的朋友一起看看吧
    2022-01-01
  • 浅谈java中的移动位运算:,>>>

    浅谈java中的移动位运算:,>>>

    这篇文章主要介绍了java中的移动位运算:,>>>文中通过示例代码介绍的非常详细,对大家的学习或者工作具有一定的参考学习价值,需要的朋友们下面随着小编来一起学习学习吧
    2019-03-03
  • JavaGUI菜单栏与文本和密码及文本域组件使用详解

    JavaGUI菜单栏与文本和密码及文本域组件使用详解

    这篇文章主要介绍了JavaGUI菜单栏与文本和密码及文本域组件使用,文中通过示例代码介绍的非常详细,对大家的学习或者工作具有一定的参考学习价值,需要的朋友们下面随着小编来一起学习吧
    2023-03-03
  • Java实现Timer的定时调度函数schedule的四种用法

    Java实现Timer的定时调度函数schedule的四种用法

    本文主要介绍了Java实现Timer的定时调度函数schedule的四种用法,文中通过示例代码介绍的非常详细,对大家的学习或者工作具有一定的参考学习价值,需要的朋友们下面随着小编来一起学习学习吧
    2023-04-04
  • Component和Configuration注解区别实例详解

    Component和Configuration注解区别实例详解

    这篇文章主要为大家介绍了Component和Configuration注解区别实例详解,有需要的朋友可以借鉴参考下,希望能够有所帮助,祝大家多多进步,早日升职加薪
    2022-11-11

最新评论