失眠网 > 爬虫实战：CentOS安装JDK 部署Java程序利用screen使其保持在前台运行

爬虫实战：CentOS安装JDK 部署Java程序利用screen使其保持在前台运行

时间：2020-04-12 23:08:42

一、CentOS 7 yum 安装与配置 JDK

参考：/github_38336924/article/details/82221258

1、检查是否已安装JDK及卸载

yum list installed | grep [java][jdk]

执行命令出现如上图所示，需要卸载，反之即不用。

卸载JAVA环境：

yum -y remove java-1.6.0-openjdk* //表时卸载所有openjdk相关文件输入yum -y remove tzdata-java.noarch //卸载tzdata-java

2、安装JDK

查看JDK软件包列表

yum search java | grep -i --color jdk

选择版本安装

yum install -y java-1.8.0-openjdk java-1.8.0-openjdk-devel# 或者如下命令，安装jdk1.8.0的所有文件yum install -y java-1.8.0-openjdk*

查看JDK是否安装成功

java -version

3、配置环境变量

JDK默认安装路径/usr/lib/jvm在/etc/profile文件添加如下命令

# set java environment JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.181-3.b13.el7_5.x86_64PATH=$PATH:$JAVA_HOME/bin CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar export JAVA_HOME CLASSPATH PATH

保存关闭profile文件，执行如下命令生效

source /etc/profile

使用如下命令，查看JDK变量

echo $JAVA_HOMEecho $PATHecho $CLASSPATH

部署Java程序

爬虫示例：（This program can increase csdn blog Page View）

import java.io.IOException;import java.io.InputStream;import .HttpURLConnection;import .URL;import java.util.*;import java.util.concurrent.*;import java.util.regex.Matcher;import java.util.regex.Pattern;/** 如果设置了自定义域名，将博客域名前缀填写入19行的变量userId中，点击运行* 本程序访问该博主名下所有博客链接*/public class UrlCrawer {static int maxPages = 20; // 填写你的博客查找页数static String userId = "hanquan";// 这里填入主页名称：例如主页为 https://hanquan./ 则填入 hanquan 即可static final String homeUrl = "https://" + userId + "./article/list/";static Set<String> urlSet = new HashSet<>();public static void getUrls() throws IOException, InterruptedException {InputStream is;String pageStr;StringBuilder curUrl = null;for (int i = 1; i < maxPages; i++) {Thread.sleep(300);System.out.println("正在查找第 " + i + " 页中的博客地址");curUrl = new StringBuilder(homeUrl);curUrl.append(i);System.out.println(curUrl);is = doGet(curUrl.toString());pageStr = inputStreamToString(is, "UTF-8");// 一整页的html源码List<String> list = getMatherSubstrs(pageStr, "(?<=href=\")https://hanquan./article/details/[0-9]{8,9}(?=\")");urlSet.addAll(list);System.out.println("加入 " + list.size() + " 个url");}}public static void main(String urlstr[]) throws IOException, InterruptedException {// ----------------------------------------------遍历每一页获取文章链接----------------------------------------------getUrls();// ---------------------------------------------------打印每个链接---------------------------------------------------System.out.println("打印每个链接");for (String s : urlSet) {System.out.println(s);}System.out.println("打印每个链接完毕");// ---------------------------------------------------多线程访问每个链接---------------------------------------------------ExecutorService executor = Executors.newCachedThreadPool();int threadCount = 5; // 并发线程数量for (int i = 0; i < threadCount; i++) {executor.execute(new MyThread(urlSet));}executor.shutdown();}public static InputStream doGet(String urlstr) throws IOException {URL url = new URL(urlstr);HttpURLConnection conn = (HttpURLConnection) url.openConnection();conn.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");InputStream inputStream = conn.getInputStream();return inputStream;}public static String inputStreamToString(InputStream is, String charset) throws IOException {byte[] bytes = new byte[1024];int byteLength = 0;StringBuffer sb = new StringBuffer();while ((byteLength = is.read(bytes)) != -1) {sb.append(new String(bytes, 0, byteLength, charset));}return sb.toString();}// 正则匹配public static List<String> getMatherSubstrs(String str, String regex) {List<String> list = new ArrayList<String>();Pattern p = pile(regex);Matcher m = p.matcher(str);while (m.find()) {list.add(m.group());}return list;}}class MyThread implements Runnable {public List<String> urlList;public MyThread(Set<String> urls) {List list = new ArrayList(urls);Collections.shuffle(list);this.urlList = list;}@Overridepublic void run() {int i = 0;while(true){for (String s : urlList) {try {Thread.sleep((int) (Math.random() * 3000));doGet(s);System.out.println(Thread.currentThread().getName() + "成功访问第" + (++i) + "个链接,共" + urlList.size() + "个:" + s);} catch (Exception e) {e.printStackTrace();}}}}public static InputStream doGet(String urlstr) throws IOException {URL url = new URL(urlstr);HttpURLConnection conn = (HttpURLConnection) url.openConnection();conn.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");InputStream inputStream = conn.getInputStream();return inputStream;}}

编译并运行

javac UrlCrawer.javajava UrlCrawer