Fix error in compiling list of dates to scrape

This commit is contained in:
paoda 2020-03-17 05:08:21 -03:00
parent 26dc65132b
commit e72b41cb71
1 changed files with 16 additions and 16 deletions

View File

@ -19,16 +19,16 @@ const WAIT = 500; // in ms
// e.g https://trendogate.com/placebydate/23424977/2015-04-01 // e.g https://trendogate.com/placebydate/23424977/2015-04-01
// Topic | Position | Date // Topic | Position | Date
const goal: Date = new Date(2015, 2, 1); // 2 is March because Month is 0-indexed (why?) const goal: Date = new Date();
let tmp : Date = new Date(); let tmp: Date = new Date(2015, 2, 1); // 2 is March because Month is 0-indexed (why?)
let days: Array<Date> = []; let days: Array<Date> = [];
while (tmp >= goal) { while (goal >= tmp) {
days.push(tmp); days.push(tmp);
tmp = addDays(tmp, -7); tmp = addDays(tmp, 7);
} }
async function scrapeData() { async function scrapeData(): Promise<void> {
const csv = new CSVManager(CSV_PATH); const csv = new CSVManager(CSV_PATH);
csv.createWriteStream(); csv.createWriteStream();
@ -63,7 +63,7 @@ function handleHttpResponse(html: string, day: Date): Array<Trend> {
} }
// SO: https://stackoverflow.com/questions/14249506/how-can-i-wait-in-node-js-javascript-l-need-to-pause-for-a-period-of-time // SO: https://stackoverflow.com/questions/14249506/how-can-i-wait-in-node-js-javascript-l-need-to-pause-for-a-period-of-time
function sleep(ms: number) { function sleep(ms: number): Promise<unknown> {
return new Promise(res => { return new Promise(res => {
setTimeout(res, ms); setTimeout(res, ms);
}); });
@ -73,7 +73,7 @@ function getNewDateUrl(date: Date): string {
return `${BASE_URL}/placebydate/${USA_ID}/${format(date, "yyyy-MM-dd")}`; return `${BASE_URL}/placebydate/${USA_ID}/${format(date, "yyyy-MM-dd")}`;
} }
function writeToCsv(trends: Array<Trend>, csv: CSVManager) { function writeToCsv(trends: Array<Trend>, csv: CSVManager): void {
for (let i = 0; i < trends.length; i++) { for (let i = 0; i < trends.length; i++) {
if (i == 20) break; // Should Only Write 20 per day hopefully. if (i == 20) break; // Should Only Write 20 per day hopefully.
@ -94,23 +94,23 @@ class CSVManager {
this.path = path; this.path = path;
} }
public write(row: string) { public write(row: string): void {
if (this.stream !== null) { if (this.stream !== null) {
this.stream.write(row, err => { if (err) throw err; }); this.stream.write(row, err => { if (err) throw err; });
} else process.stderr.write(`Unable to write to "${this.path}". Stream does not exist.`); } else process.stderr.write(`Unable to write to "${this.path}". Stream does not exist.`);
} }
public createWriteStream() { public createWriteStream(): void {
if (!fs.existsSync(this.path)) fs.closeSync(fs.openSync(this.path, "w")); if (!fs.existsSync(this.path)) fs.closeSync(fs.openSync(this.path, "w"));
this.stream = fs.createWriteStream(this.path, { flags: "a" }); this.stream = fs.createWriteStream(this.path, { flags: "a" });
} }
public closeWriteStream() { public closeWriteStream(): void {
this.stream.end(); this.stream.end();
this.stream = null; this.stream = null;
} }
public getPath() { public getPath(): string {
return this.path; return this.path;
} }
} }
@ -121,24 +121,24 @@ class Trend {
private ranking: number; private ranking: number;
constructor(ranking: number, name: string, date: Date) { constructor(ranking: number, name: string, date: Date) {
this.name = name; this.name = name.trim();
this.date = date; this.date = date;
this.ranking = ranking; this.ranking = ranking;
} }
public getName() { public getName(): string {
return this.name; return this.name;
} }
public getDate() { public getDate(): Date {
return this.date; return this.date;
} }
public getRanking() { public getRanking(): number {
return this.ranking; return this.ranking;
} }
public toCsv() { public toCsv(): string {
return `${this.name}, ${this.ranking}, ${format(this.date, "yyyy-MM-dd")}\n`; return `${this.name}, ${this.ranking}, ${format(this.date, "yyyy-MM-dd")}\n`;
} }
} }