paoda
/
trendogate_scraper
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated Constraints of the Scraper 

No longer just the past 200 or so days. Now the scraper gets all
trending data from 2015-03-01 to <current_day> once a week.
This commit is contained in:
paoda 2020-03-17 04:38:30 -03:00
parent 9c824bfdd6
commit 26dc65132b
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
index.ts
View File

@ -4,6 +4,7 @@
* @license MIT
* @version 2020.03.17
*/
import { addDays, format } from 'date-fns';
import cheerio from 'cheerio';
import axios from 'axios';
@ -18,13 +19,13 @@ const WAIT = 500; // in ms
// e.g https://trendogate.com/placebydate/23424977/2015-04-01
// Topic | Position | Date
const now: Date = new Date();
let tmp : Date = now;
const goal: Date = new Date(2015, 2, 1); // 2 is March because Month is 0-indexed (why?)
let tmp : Date = new Date();
let days: Array<Date> = [];
for (let i = 0; i < NUMBER_OF_DAYS; i++) {
tmp = addDays(tmp, -1);
while (tmp >= goal) {
days.push(tmp);
tmp = addDays(tmp, -7);
}
async function scrapeData() {