liweichun
/
my-bookmark
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

增加定时10分钟抓取文章

This commit is contained in:
HelloWorld 2020-04-14 09:49:47 +08:00
parent 18fd732daa
commit c6d8a76102
3 changed files with 46 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -13,6 +13,7 @@
"cheerio": "^1.0.0-rc.3",
"fs-extra": "^9.0.0",
"node-readability": "^3.0.0",
"request": "^2.88.2",
"think-cache": "^1.0.0",
"think-cache-file": "^1.0.8",
"think-logger3": "^1.0.0",

42
src/config/crontab.js Normal file
View File

@ -0,0 +1,42 @@
const request = require("request");
module.exports = [{
interval: 1000 * 60 * 10, // 10分钟抓取一次
immediate: true,
handle: async () => {
let form = {
userId: null,
lastupdataTime: new Date().getTime(),
pageNo: 1,
pageSize: 1000,
sort: "desc",
renderType: 0,
date: think.datetime(new Date(), "YYYY年MM月DD日"),
idfa: "d4995f8a0c9b2ad9182369016e376278",
os: "ios",
osv: "9.3.5"
};
const url = "https://api.shouqu.me/api_service/api/v1/daily/dailyMark";
request.post({ url, form }, async function (error, response, body) {
if (!error && response && response.statusCode == 200) {
let data = JSON.parse(body).data;
let list = data.list;
let dataList = [];
for (const item of list) {
dataList.push({
id: item.articleId,
title: item.title,
url: item.url,
clickCount: item.favCount,
tagName: item.sourceName,
createdAt: think.datetime(item.updatetime > item.createtime ? item.createtime : item.updatetime),
lastClick: think.datetime(item.updatetime <= item.createtime ? item.createtime : item.updatetime),
snap: item.images_upd,
icon: item.sourceLogo
})
}
await think.model('hot_bookmarks').addMany(dataList, { replace: true });
}
})
}
}]

6
update.sql
View File

@ -44,10 +44,10 @@ CHANGE COLUMN `fav_count` `clickCount` smallint(6) NULL DEFAULT 1 AFTER `url`,
CHANGE COLUMN `created_by` `tagName` varchar(32) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL AFTER `clickCount`,
CHANGE COLUMN `snap_url` `snap` varchar(1024) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL AFTER `last_click`,
CHANGE COLUMN `favicon_url` `icon` varchar(1024) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL AFTER `snap`,
ADD COLUMN `createAt` datetime NULL AFTER `tagName`,
ADD COLUMN `lastClick` datetime NULL AFTER `createAt`,
ADD COLUMN `createdAt` datetime NULL AFTER `tagName`,
ADD COLUMN `lastClick` datetime NULL AFTER `createdAt`,
MODIFY COLUMN `id` int(11) NOT NULL AUTO_INCREMENT FIRST;
UPDATE hot_bookmarks SET createAt = FROM_UNIXTIME(created_at/1000), lastClick = FROM_UNIXTIME(last_click/1000);
UPDATE hot_bookmarks SET createdAt = FROM_UNIXTIME(created_at/1000), lastClick = FROM_UNIXTIME(last_click/1000);
ALTER TABLE `hot_bookmarks`
DROP COLUMN `created_at`,
DROP COLUMN `last_click`;