package com.ssm.service.task;
import java.awt.Image;
import java.awt.image.RenderedImage;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.imageio.ImageIO;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import com.ssm.mapper.NewsMapper;
import com.ssm.po.News;
import com.ssm.utils.EncoderUtil;
import com.ssm.utils.FileUtil;
import com.ssm.utils.HttpUtil;
@Service("spider")
public class Spider {
@Autowired
NewsMapper newsMapper;
@Scheduled(fixedRate = 1000 * 1000)
public void getNotices() throws Exception {
List<News> newsList = getComputerNews();
News news = getDragonNews();
newsMapper.deleteAllNews();
if (news.getTitle() != null) {
newsMapper.insertNews(news);
}
newsMapper.insertNewsList(newsList);
}
public News getDragonNews() {
News news = new News();
int fileCount = 0;
// 获取标题
String dsUrl = "http://cs.scu.edu.cn/cs/xyxw/H9501index_1.htm";
dsUrl = HttpUtil.sendGet(dsUrl, "GBK");
Pattern pattern = Pattern
.compile("<A href=http://222.178.203.72:19005/whst/63/_bac1.05zrbtzdctzbm//ds2016 target=_blank>(.+?)</A>");
Matcher matcher = pattern.matcher(dsUrl);
if (matcher.find()) {
news.setTitle(matcher.group(1));
}
// 获取时间
pattern = Pattern
.compile("<DIV align=right><FONT size=2>.(.+?).</FONT>");
matcher = pattern.matcher(dsUrl);
if (matcher.find()) {// 设置日期
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");// 设置日期格式
try {
// 将字符串转换成date类型
Date date = format.parse(matcher.group(1));
news.setTime(date);
System.out.println(date);
} catch (ParseException e) {
System.out.println("日期格式不正确");
e.printStackTrace();
}
}
// 获取内容
String dsUrl2 = "http://cbd2016.scu.edu.cn/ds2016/";
dsUrl2 = HttpUtil.sendGet(dsUrl2, "UTF-8");
pattern = Pattern
.compile("<span style=\"font-size: 18px;\"><span style=\"font-weight: bold;\">.+?</span><br>"
+ "</p><p style=\"margin-top: 0px; margin-bottom: 0px; word-spacing:"
+ " 0px; text-align: justify;\">.+?<span style=\"font-size: 18px; font-family:.+?</span>"
+ "</p><p style=\"margin-top: 0px; margin-bottom: 0px; word-spacing: 0px; text-align: justify;\">");
matcher = pattern.matcher(dsUrl2);
if (matcher.find()) {
System.out.println(matcher.group(0));
String contentStr = matcher.group(0).replaceAll(
"images/slide10.jpg", "http://cbd2016.scu.edu.cn/ds2016/");
// 添加页面头
contentStr = contentStr
.replace(
contentStr,
"<html><head> <base target=\"_blank\" /> <meta"
+ " http-equiv=\"Content-Type\" content=\"textml; charset=utf-8\"/>"
+ contentStr + "<ml>");
// 获取图片
pattern = Pattern.compile("images/slide.+?.jpg");
matcher = pattern.matcher(dsUrl2);
if (matcher.find()) {
news.setPic("http://cbd2016.scu.edu.cn/ds2016/"
+ matcher.group(0));
}
// 获取相对路径
String relativelyPath = System.getProperty("b2cweb.root");
relativelyPath = relativelyPath.replace("\\", "/");
int index = relativelyPath.indexOf(".");
relativelyPath = relativelyPath.substring(0, index);
String filePaths = relativelyPath
+ "SCU_News_Notice/WebContent/html/computer/news/"
+ fileCount + ".html";
// System.out.println(contentStr);
FileUtil.writeIntoFile(contentStr, filePaths, false);
news.setContent("http://localhost:8080/SCU_News_Notice/html/computer/news/"
+ fileCount + ".html");
}
// news.setContent("龙星计划");
news.setAddress("http://cbd2016.scu.edu.cn/ds2016/");
news.setAcademyId(1);
return news;
}
public List<News> getComputerNews() {
// 存放新闻详细信息页
List<String> links = new ArrayList<String>();
int count = 1;
// http://cs.scu.edu.cn/cs/xytz/H9502index_1.htm
String htmlStr = HttpUtil.sendGet(
"http://cs.scu.edu.cn/cs/xyxw/H9501index_" + count + ".htm",
"GBK");
// 转换成utf-8编码格式
// htmlStr = EncoderUtil.getUTF8StringFromGBKString(htmlStr);
// 使用正则表达式匹配详细地址的url
Pattern pattern = Pattern
.compile("<A href=http://222.178.203.72:19005/whst/63/=cnvmknZczbrcmzmds//cs/xyxw/webinfo(.+?) target=_blank>");
Matcher matcher = pattern.matcher(htmlStr);
boolean isFind = matcher.find();
while (isFind) {// 如果有匹配的通知详细地址，继续循环
while (matcher.find()) {
links.add("http://cs.scu.edu.cn/cs/xyxw/webinfo"
+ matcher.group(1));
System.out.println("text" + matcher.group(1));
}
count++;
htmlStr = HttpUtil
.sendGet("http://cs.scu.edu.cn/cs/xyxw/H9501index_" + count
+ ".htm", "GBK");
// htmlStr = EncoderUtil.getUTF8StringFromGBKString(htmlStr);
matcher = pattern.matcher(htmlStr);
isFind = matcher.find();
}
// 存放通知具体信息
List<News> newsList = new ArrayList<News>();
int fileCount = 1;
for (String link : links) {
News news = new News();
htmlStr = HttpUtil.sendGet(link, "GBK");
htmlStr = EncoderUtil.getUTF8StringFromGBKString(htmlStr);
// 获取标题
pattern = Pattern.compile("<DIV align=center> (.+?)</DIV>");
matcher = pattern.matcher(htmlStr);
if (matcher.find()) {// 设置标题
news.setTitle(matcher.group(1));
}
// 获取日期
pattern = Pattern.compile("</SPAN> ([0-9].+?)<SPAN class=hangjc "
+ "style=\"LINE-HEIGHT: 30px\" valign=\"bottom\">");
matcher = pattern.matcher(htmlStr);
if (matcher.find()) {// 设置日期
SimpleDateFormat format = new SimpleDateFormat(
"yyyy-MM-dd HH:mm");// 设置日期格式
try {
// 将字符串转换成date类型
Date date = format.parse(matcher.group(1));
news.setTime(date);
System.out.println(date);
} catch (ParseException e) {
System.out.println("日期格式不正确");
e.printStackTrace();
}
}
// 获取内容
Document doc = Jsoup.parse(htmlStr);
Element contentEle = doc.getElementById("BodyLabel");
String contentStr = contentEle.toString();
// 添加图片的完整路径
contentStr = contentStr.replaceAll("src=\"",
"src=\"http://cs.scu.edu.cn");
// 添加页面头
contentStr = contentStr
.replace(
contentStr,
"<html><head> <base target=\"_blank\" /> <meta"
+ " http-equiv=\"Content-Type\" content=\"textml; charset=utf-8\"/>"
+ contentStr + "<ml>");
// 获取相对路径
String relativelyPath = System.getProperty("b2cweb.root");
relativelyPath = relativelyPath.replace("\\", "/");
int index = relativelyPath.indexOf(".");
relativelyPath = relativelyPath.substring(0, index);
String filePaths = relativelyPath
+ "SCU_News_Notice/WebContent/html/computer/news/"
+ fileCount + ".html";
// System.out.println(contentStr);
FileUtil.writeIntoFile(contentStr, filePaths, false);
news.setContent("http://localhost:8080/SCU_News_Notice/html/computer/news/"
+ fileCount + ".html");
// 获取图片
Elements images = contentEle.getElementsByTag("img");
String[] imageUrls = new String[images.size()];
for (int i = 0; i < imageUrls.length; i++) {
imageUrls[i] = images.get(i).attr("src");
news.setPic(imageUrls[i]);
}
news.setAcademyId(1);
news.setAddress(link);
newsList.add(news);
fileCount++;
}
System.out.println(links.size());
System.out.println(newsList.size());
return newsList;
}
}
Java实现网路爬虫爬取新闻信息 （482个子文件） 
Spider.class 8KB MyBatisDao.class 4KB HttpUtil.class 4KB NewsServiceImpl.class 2KB NewsController.class 2KB News.class 2KB FileUtil.class 2KB Notice.class 1KB EncoderUtil.class 1KB CustomDateConverter.class 1KB Academy.class 711B NewsService.class 618B NewsMapper.class 615B AcademyMapper.class 122B NoticeMapper.class 120B .classpath 841B org.eclipse.wst.common.component 497B org.eclipse.wst.jsdt.ui.superType.container 49B
287.html 29KB 1.html 29KB 234.html 26KB 320.html 26KB 176.html 21KB 8.html 19KB 116.html 19KB 91.html 19KB 73.html 17KB 211.html 17KB 7.html 17KB 221.html 16KB 106.html 16KB 278.html 16KB 182.html 16KB 16.html 16KB 105.html 15KB 197.html 15KB 103.html 15KB 22.html 15KB 75.html 15KB 283.html 14KB 185.html 14KB 191.html 14KB 168.html 14KB 61.html 14KB 68.html 13KB 72.html 13KB 358.html 13KB 351.html 12KB 3.html 12KB 282.html 12KB 164.html 12KB 224.html 12KB 135.html 12KB 186.html 12KB 134.html 12KB 355.html 12KB 260.html 12KB 31.html 12KB 162.html 12KB 112.html 12KB 202.html 12KB 285.html 11KB 297.html 11KB 156.html 11KB 109.html 11KB 269.html 11KB 361.html 11KB 179.html 11KB 130.html 11KB 96.html 11KB 124.html 11KB 69.html 11KB 146.html 11KB 147.html 11KB 175.html 11KB 210.html 11KB 13.html 10KB 326.html 10KB 343.html 10KB 108.html 10KB 102.html 10KB 9.html 10KB 95.html 10KB 171.html 10KB 341.html 10KB 133.html 10KB 66.html 10KB 25.html 10KB 273.html 10KB 138.html 10KB 11.html 10KB 336.html 10KB 128.html 9KB 346.html 9KB 227.html 9KB 342.html 9KB 289.html 9KB 33.html 9KB 379.html 9KB 154.html 9KB共 482 条
- 1
- 2
- 3
- 4
- 5
资源评论
凌风1205
- 粉丝: 2
- 资源: 24
最新资源
- DeepSeek模型优势：算力、成本角度解读.pdf
- DeepSeek手册2 -DeepSeek大模型赋能高校教学和科研.pdf
- 多功能AI助手Manus的使用指南与最佳实践-提升工作效率、创意和数据处理
- DeepSeek手册3 -DeepSeek大模型及其企业应用实践：企业人员的大模型宝典.pdf
- DeepSeek企业落地应用讲义精华全版 .pdf
- DeepSeek私有化部署和一体机.pdf
- DeepSeek手册4 -DeepSeek大模型赋能政府数字化转型.pdf
- DeepSeek完全实用手册-从技术原理到使用技巧.pdf
- 西门子1200PLC 自动贴标机 三轴伺服控制 包括PLC程序和触摸屏程序 1触摸屏是西门子TP900 23轴伺服PTO，脉冲加方向控制 3梯形图和SCL编写 红圆多种执行器：机械手
- Deepseek引领技术潮流央国企应重新规划大模型算力投入和应用节奏.pdf
- DeepSeek原理和落地应用.pdf
- DeepSeek系列-提示词工程和落地场景.pdf
- DeepSeek在金融银行的应用方案.pdf
- PyCharm IDE安装配置指南：涵盖多平台详细步骤、环境搭建及项目创建
- 从DeepSeek爆火 看25年AI行业发展 .pdf
- DeepSeek在医院业务系统中的应用.pdf
资源上传下载、课程学习等过程中有任何疑问或建议，欢迎提出宝贵意见哦~我们会及时处理！
点击此处反馈
