paoda
/
trendogate_scraper
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed Formatting of Date

This commit is contained in:
paoda 2020-03-05 15:11:51 -04:00
parent 868477a9ae
commit 0e9710e545
1 changed files with 5 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
index.ts
View File

@ -1,6 +1,6 @@
"use strict"; "use strict";
import { addDays } from 'date-fns'; import { addDays, format } from 'date-fns';
import cheerio from 'cheerio'; import cheerio from 'cheerio';
import axios from 'axios'; import axios from 'axios';
import fs from 'fs'; import fs from 'fs';
@ -9,7 +9,7 @@ const BASE_URL = "https://trendogate.com/";
const CSV_PATH = "./data.csv"; const CSV_PATH = "./data.csv";
const USA_ID = 23424977; const USA_ID = 23424977;
const NUMBER_OF_DAYS = 100; const NUMBER_OF_DAYS = 100;
const WAIT = 5000; // in ms const WAIT = 2000; // in ms
// e.g https://trendogate.com/placebydate/23424977/2015-04-01 // e.g https://trendogate.com/placebydate/23424977/2015-04-01
// Topic | Position | Date // Topic | Position | Date
@ -64,19 +64,7 @@ function sleep(ms: number) {
} }
function getNewDateUrl(date: Date): string { function getNewDateUrl(date: Date): string {
const year: number = date.getFullYear(); return `${BASE_URL}/placebydate/${USA_ID}/${format(date, "yyyy-MM-dd")}`;
const month: number = date.getMonth();
const day: number = date.getDay();
let monthStr: string;
let dayStr: string;
if (month < 10) monthStr = `0${month}`;
else monthStr = `${month}`;
if (day < 10) dayStr = `0${day}`;
else dayStr = `${day}`;
return `${BASE_URL}/placebydate/${USA_ID}/${year}-${monthStr}-${dayStr}`;
} }
function writeToCsv(trends: Array<Trend>, csv: CSVManager) { function writeToCsv(trends: Array<Trend>, csv: CSVManager) {
@ -100,7 +88,7 @@ class CSVManager {
} }
public createWriteStream() { public createWriteStream() {
if (fs.existsSync(this.path)) fs.closeSync(fs.openSync(this.path, "w")); if (!fs.existsSync(this.path)) fs.closeSync(fs.openSync(this.path, "w"));
this.stream = fs.createWriteStream(this.path, { flags: "a" }); this.stream = fs.createWriteStream(this.path, { flags: "a" });
} }
@ -114,7 +102,6 @@ class CSVManager {
} }
} }
class Trend { class Trend {
private name: string; private name: string;
private date: Date; private date: Date;
@ -139,9 +126,8 @@ class Trend {
} }
public toCsv() { public toCsv() {
return `${this.name}, ${this.ranking}, ${this.date}\n`; return `${this.name}, ${this.ranking}, ${format(this.date, "yyyy-MM-dd")}\n`;
} }
} }
scrapeData(); scrapeData();