完成部分抓取热门文章

app.js

@@ -78,5 +78,6 @@ app.use(function(err, req, res, next) {
 api.checkSnapFaviconState();
 api.getSnapByTimer();
 api.getFaviconByTimer();
+api.getHotBookmarks();
 
 module.exports = app;

database/db.js

@@ -775,4 +775,30 @@ db.updateBookmarkFaviconState = function(id, faviconState) {
     });
 }
 
+db.addHotBookmark = function(bookmark) {
+    var insertSql = "INSERT INTO `hot_bookmarks` (`id`, `date`, `title`, `url`, `fav_count`, `created_by`, `created_at`, `last_click`, `snap_url`, `favicon_url`) VALUES ('" + client.escape(bookmark.id) + "', '" + client.escape(bookmark.date) + "', '" + client.escape(bookmark.title) + "', '" + client.escape(bookmark.url) + "', '" + client.escape(bookmark.fav_count) + "', '" + client.escape(bookmark.created_by) + "', '" + client.escape(bookmark.created_at) + "', '" + client.escape(bookmark.last_click) + "', '" + client.escape(bookmark.snap_url) + "', '" + client.escape(bookmark.favicon_url) + "')";
+
+    var selectSql = "SELECT * FROM `bookmarks` WHERE `id` = '" + bookmark.id + "' OR `url` = '" + bookmark.url + "'";
+    console.log(insertSql, selectSql);
+    return new Promise(function(resolve, reject) {
+        client.query(selectSql, (err, result) => {
+            if (err) {
+                reject(err);
+            } else {
+                if (result.length >= 1) {
+                    resolve(result[0].id);
+                } else {
+                    client.query(insertSql, (err, result) => {
+                        if (err) {
+                            reject(err);
+                        } else {
+                            resolve(result.insertId);
+                        }
+                    });
+                }
+            }
+        });
+    });
+};
+
 module.exports = db;

package.json

@@ -21,6 +21,7 @@
     "multer": "^1.3.0",
     "mysql": "^2.11.1",
     "node-readability": "^2.2.0",
+    "request": "^2.81.0",
     "serve-favicon": "~2.3.0",
     "supervisor": "^0.11.0",
     "webshot": "^0.18.0"

routes/api.js

@@ -9,6 +9,7 @@ var multer = require('multer');
 var webshot = require('webshot');
 var fs = require('fs');
 var favicon = require('favicon');
+var request = require('request');
 
 var storage = multer.diskStorage({
     destination: function(req, file, cb) {
@@ -1006,6 +1007,73 @@ api.getFaviconByTimer = function() {
     }, timeout);
 }
 
+api.getHotBookmarks = function() {
+    console.log('getHotBookmarks...........');
+    var timeout = 1000 * 20;
+    var busy = false;
+    setInterval(function() {
+        if (busy) {
+            console.log('getHotBookmarks is busy')
+            return;
+        }
+        busy = true;
+        var today = new Date();
+        var requireData = {
+            idfa: "d4995f8a0c9b2ad9182369016e376278",
+            os: "ios",
+            osv: "9.3.5",
+            userId: null,
+            lastupdataTime: new Date().getTime(),
+            pageNo: 1,
+            pageSize: 1000,
+            sort: 'desc',
+            renderType: 0,
+            date: CurentDate(0),
+        }
+        var url = "https://api.shouqu.me/api_service/api/v1/daily/dailyMark";
+        var alterRex = "/mmbiz.qpic.cn|images.jianshu.io|zhimg.com/g";
+        var defaultSnap = "./images/snap/default.png";
+        var defaultFavicon = "./images/favicon/default.ico";
+        request.post({
+            url: url,
+            form: requireData
+        }, function(error, response, body) {
+            if (response && response.statusCode == 200) {
+                var data = JSON.parse(body).data;
+                var bookmarks = [];
+                data.list.forEach((b) => {
+                    var bookmark = {};
+                    bookmark.id = b.articleId;
+                    bookmark.date = parseInt(today.format('yyyyMMdd'));
+                    bookmark.title = b.title;
+                    bookmark.url = b.url;
+                    bookmark.fav_count = b.favCount;
+                    bookmark.created_by = b.sourceName;
+                    bookmark.created_at = b.updatetime > b.createtime ? b.createtime : b.updatetime;
+                    bookmark.last_click = b.updatetime < b.createtime ? b.createtime : b.updatetime;
+                    if (b.imageList.length >= 1) {
+                        bookmark.snap_url = (data.pageNo == 1 ? (b.imageList[0].url.match(alterRex) != null ? defaultSnap : b.imageList[0].url) : defaultSnap);
+                    } else {
+                        bookmark.snap_url = defaultSnap;
+                    }
+                    bookmark.favicon_url = b.sourceLogo || defaultFavicon;
+                    bookmarks.push(bookmark);
+                    if (bookmarks.length == 1) {
+                        db.addHotBookmark(bookmark)
+                            .then((id) => {
+                                console.log(id);
+                            })
+                            .catch((err) => {
+                                console.log('getHotBookmarks err', err);
+                            });
+                    }
+
+                })
+            }
+        });
+    }, timeout);
+}
+
 function md5(str) {
     return crypto.createHash('md5').update(str).digest('hex');
 };
@@ -1020,4 +1088,17 @@ function copyFile(sourceFile, destFile) {
     });
 }
 
+function CurentDate(i) {
+    if (i == undefined) {
+        i = 0;
+    }
+    var now = new Date();
+    now.setTime(now.getTime() + i * 24 * 60 * 60 * 1000);
+    var year = now.getFullYear(); //年
+    var month = now.getMonth() + 1; //月
+    var day = now.getDate(); //日
+    var clock = year + "年" + month + "月" + day + "日";
+    return (clock);
+}
+
 module.exports = api;

schema.sql

@@ -67,10 +67,11 @@ CREATE TABLE `advices` (
 drop table if exists hot_bookmarks;
 CREATE TABLE `hot_bookmarks` (
   `id` int(11) NOT NULL AUTO_INCREMENT,     -- id(articleId)
+  `date` int(11) NOT NULL DEFAULT 0,        -- 日期(自己添加)
   `title` varchar(255) DEFAULT NULL,        -- 标题(title)
   `description` varchar(4096) DEFAULT NULL, -- 描述(自己添加)
   `url` varchar(1024) DEFAULT NULL,         -- 链接(url)
-  `click_count` smallint DEFAULT 1,         -- 总共点击次数(favCount)
+  `fav_count` smallint DEFAULT 1,           -- 总共收藏人数(favCount)
   `created_by` varchar(64) DEFAULT NULL,    -- 创建者(sourceName)
   `created_at` bigint DEFAULT 0,            -- 创建时间(updatetime)
   `last_click` bigint DEFAULT 0,            -- 最后一次点击时间(createtime)