同时下载多个文件不过单文件没有多线程的下载并且没有断点续传功能继续完善:
view plaincopy to clipboardprint?
package comutilfile;
public class Files {
/***
* 获取应用程序的根目录
* @return 应用程序根目录
*/
public static String getSysPath(){
return SystemgetProperty(userdir);
}
}
package comutilfile;
public class Files {
/***
* 获取应用程序的根目录
* @return 应用程序根目录
*/
public static String getSysPath(){
return SystemgetProperty(userdir);
}
}view plaincopy to clipboardprint?
view plaincopy to clipboardprint?
<PRE class=csharp name=code>package recrawl;
import javaioFile;
import javaioFileOutputStream;
import javaioIOException;
import javaioInputStream;
import HttpURLConnection;
import URL;
import rehttpHttp;
public class WebSpider implements Runnable{
private Http http = new Http();
private String webAddress = ;
private String destFile = ;
public void setWebAddress(String webAddress){
thiswebAddress = webAddress;
}
public void setDestFile (String destFile){
thisdestFile = destFile;
}
public boolean download() throws IOException InterruptedException {
HttpURLConnection httpConn = null;
try {
URL url = new URL(webAddress);
httpConn = (HttpURLConnection) urlopenConnection();
(GET);
(UserAgent Mozilla/ (Windows; U; Windows NT ; zhCN; rv:) Gecko/ Firefox/);
InputStream in = ();
String fileType = httpfileType(());
Systemoutprintln(fileType);
FileOutputStream out = new FileOutputStream(new File(destFile + fileType));
int chByte = inread();
while (chByte != ) {
outwrite(chByte);
//Systemoutprintln(chByte);
chByte = inread();
}
} catch (Exception ex) {
Systemoutprintln(extoString());
} finally {
();
}
return true;
}
public void run() {
try {
//Systemoutprintln(ThreadcurrentThread()getName());
download();
} catch (IOException e) {
eprintStackTrace();
} catch (InterruptedException e) {
eprintStackTrace();
}
}
}
</PRE>
view plaincopy to clipboardprint?package recrawl; import javaioFile; import javaioFileOutputStream; import javaioIOException; import javaioInputStream; import HttpURLConnection; import URL; import rehttpHttp; public class WebSpider implements Runnable{ private Http http = new Http(); private String webAddress = ; private String destFile = ; public void setWebAddress(String webAddress){ thiswebAddress = webAddress; } public void setDestFile (String destFile){ thisdestFile = destFile; } public boolean download() throws IOException InterruptedException { HttpURLConnection httpConn = null; try { URL url = new URL(webAddress); httpConn = (HttpURLConnection) urlopenConnection(); (GET); (UserAgent Mozilla/ (Windows; U; Windows NT ; zhCN; rv:) Gecko/ Firefox/); InputStream in = (); String fileType = httpfileType(()); Systemoutprintln(fileType); FileOutputStream out = new FileOutputStream(new File(destFile + fileType)); int chByte = inread(); while (chByte != ) { outwrite(chByte); //Systemoutprintln(chByte); chByte = inread(); } } catch (Exception ex) { Systemoutprintln(extoString()); } finally { (); } return true; } public void run() { try { //Systemoutprintln(ThreadcurrentThread()getName()); download(); } catch (IOException e) { eprintStackTrace(); } catch (InterruptedException e) { eprintStackTrace(); } } } package recrawl;
import javaioFile;
import javaioFileOutputStream;
import javaioIOException;
import javaioInputStream;
import HttpURLConnection;
import URL;
import rehttpHttp;
public class WebSpider implements Runnable{
private Http http = new Http();
private String webAddress = ;
private String destFile = ;
public void setWebAddress(String webAddress){
thiswebAddress = webAddress;
}
public void setDestFile (String destFile){
thisdestFile = destFile;
}
public boolean download() throws IOException InterruptedException {
HttpURLConnection httpConn = null;
try {
URL url = new URL(webAddress);
httpConn = (HttpURLConnection) urlopenConnection();
(GET);
(UserAgent Mozilla/ (Windows; U; Windows NT ; zhCN; rv:) Gecko/ Firefox/);
InputStream in = ();
String fileType = httpfileType(());
Systemoutprintln(fileType);
FileOutputStream out = new FileOutputStream(new File(destFile + fileType));
int chByte = inread();
while (chByte != ) {
outwrite(chByte);
//Systemoutprintln(chByte);
chByte = inread();
}
} catch (Exception ex) {
Systemoutprintln(extoString());
} finally {
();
}
return true;
}
public void run() {
try {
//Systemoutprintln(ThreadcurrentThread()getName());
download();
} catch (IOException e) {
eprintStackTrace();
} catch (InterruptedException e) {
eprintStackTrace();
}
}
}
view plaincopy to clipboardprint?
<PRE class=csharp name=code>package recrawl;
import javaioIOException;
import comutilfileFiles;
public class Crawl {
/**
* @param args
* @throws IOException
* @throws InterruptedException
*/
public static void main(String[] args) throws IOException InterruptedException {
long begin = SystemcurrentTimeMillis();
WebSpider spider = new WebSpider();
WebSpider spider = new WebSpider();
spidersetWebAddress();
spidersetDestFile(FilesgetSysPath() + /+spider);
spidersetWebAddress();
spidersetDestFile(FilesgetSysPath() + /+spider);
Thread t = new Thread(spider);
Thread t = new Thread(spider);
tstart();
tstart();
tjoin();
tjoin();
Systemoutprintln(the end);
Systemoutprintln(SystemcurrentTimeMillis() begin);
}
}</PRE>
<PRE class=csharp name=code> </PRE>
<PRE class=csharp name=code>测试通过</PRE>
<PRE class=csharp name=code></PRE>
