diff --git a/index.ts b/index.ts index ceb3ce9..38bf67a 100644 --- a/index.ts +++ b/index.ts @@ -8,13 +8,13 @@ import fs from 'fs'; const BASE_URL = "https://trendogate.com/"; const CSV_PATH = "./data.csv"; const USA_ID = 23424977; -const NUMBER_OF_DAYS = 100; -const WAIT = 2000; // in ms +const NUMBER_OF_DAYS = 200; +const WAIT = 500; // in ms // e.g https://trendogate.com/placebydate/23424977/2015-04-01 // Topic | Position | Date -const now: Date = new Date(); +const now: Date = new Date(); let tmp : Date = now; let days: Array = []; @@ -51,6 +51,7 @@ function handleHttpResponse(html: string, day: Date): Array { $('div.panel > ul.list-group').children().each((i, child) => { let term = child.firstChild.firstChild.data; trends.push(new Trend(i + 1, term, day)); + }); return trends; @@ -69,7 +70,14 @@ function getNewDateUrl(date: Date): string { function writeToCsv(trends: Array, csv: CSVManager) { for (let i = 0; i < trends.length; i++) { - csv.write(trends[i].toCsv()); + if (i == 20) break; // Should Only Write 20 per day hopefully. + + let trend = trends[i]; + + // Test if The String is Latin+ Some Other Codepoint ranges. + if (trend.getName().match(/^([A-Za-z\u00C0-\u00D6\u00D8-\u00f6\u00f8-\u00ff\s\#]*)$/i)) { + csv.write(trends[i].toCsv()); + } } }