paoda
/
trendogate_scraper
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated Constraints of the Scraper 

No longer just the past 200 or so days. Now the scraper gets all
trending data from 2015-03-01 to <current_day> once a week.
This commit is contained in:
paoda 2020-03-17 04:38:30 -03:00
parent 9c824bfdd6
commit 26dc65132b
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
index.ts
View File

@ -4,6 +4,7 @@
* @license MIT * @license MIT
* @version 2020.03.17 * @version 2020.03.17
*/ */
import { addDays, format } from 'date-fns'; import { addDays, format } from 'date-fns';
import cheerio from 'cheerio'; import cheerio from 'cheerio';
import axios from 'axios'; import axios from 'axios';
@ -18,13 +19,13 @@ const WAIT = 500; // in ms
// e.g https://trendogate.com/placebydate/23424977/2015-04-01 // e.g https://trendogate.com/placebydate/23424977/2015-04-01
// Topic | Position | Date // Topic | Position | Date
const now: Date = new Date(); const goal: Date = new Date(2015, 2, 1); // 2 is March because Month is 0-indexed (why?)
let tmp : Date = now; let tmp : Date = new Date();
let days: Array<Date> = []; let days: Array<Date> = [];
for (let i = 0; i < NUMBER_OF_DAYS; i++) { while (tmp >= goal) {
tmp = addDays(tmp, -1);
days.push(tmp); days.push(tmp);
tmp = addDays(tmp, -7);
} }
async function scrapeData() { async function scrapeData() {