paoda
/
trendogate_scraper
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add filter for latin chracters

This commit is contained in:
paoda 2020-03-06 15:09:56 -04:00
parent 0e9710e545
commit 724af523ff
1 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
index.ts
View File

@ -8,8 +8,8 @@ import fs from 'fs';
const BASE_URL = "https://trendogate.com/";
const CSV_PATH = "./data.csv";
const USA_ID = 23424977;
const NUMBER_OF_DAYS = 100;
const WAIT = 2000; // in ms
const NUMBER_OF_DAYS = 200;
const WAIT = 500; // in ms
// e.g https://trendogate.com/placebydate/23424977/2015-04-01
// Topic | Position | Date
@ -51,6 +51,7 @@ function handleHttpResponse(html: string, day: Date): Array<Trend> {
$('div.panel > ul.list-group').children().each((i, child) => {
let term = child.firstChild.firstChild.data;
trends.push(new Trend(i + 1, term, day));
});
return trends;
@ -69,9 +70,16 @@ function getNewDateUrl(date: Date): string {
function writeToCsv(trends: Array<Trend>, csv: CSVManager) {
for (let i = 0; i < trends.length; i++) {
if (i == 20) break; // Should Only Write 20 per day hopefully.
let trend = trends[i];
// Test if The String is Latin+ Some Other Codepoint ranges.
if (trend.getName().match(/^([A-Za-z\u00C0-\u00D6\u00D8-\u00f6\u00f8-\u00ff\s\#]*)$/i)) {
csv.write(trends[i].toCsv());
}
}
}
class CSVManager {
private stream: fs.WriteStream = null;