bin/loadTweets.js
/*
 *
 *Function to read in a tweet input file and create relevant GeoJSON data
 *
 */
Function to read in a tweet input file and create relevant GeoJSON data
Function to read in a tweet input file and create relevant GeoJSON data
function processFile(fname) {
	console.log('Processing file: '+fname);
	reader.eachLine(fname, function(line, last) {
		if (line.substring(line.length - 2) == '}}') { 
			var tTweet, geojson;
			lines++; 
			try {
				tTweet = JSON.parse(line); 
				var turf = tweetAsGeoJson(tTweet); 
				if (turf) { 
					turf.properties.id = tTweet.id_str;
					turf.properties.text = tTweet.text;
					turf.properties.user_id = tTweet.user.id_str;
					turf.properties.screen_name = tTweet.user.screen_name;
					if (tTweet.extended_entities && tTweet.extended_entities.media && tTweet.extended_entities.media[0].media_url_https ) {	
						turf.properties.media = tTweet.extended_entities.media[0].media_url_https;
					}
					else {
						turf.properties.media = "Image Not Found";
					}
					findState(turf.properties.country,turf);
					if(turf.properties.state == 'Not Found') {
						stateNotFound++;
					}
					if(turf.properties.country == 'Not Found') {
						turf.properties.country = findCountry(turf); 
					}
					if (turf.properties.country != 'Not Found') { 
						turf.properties.timezone = findTimeZone(turf); 
						if (turf.properties.timezone == 'Antarctica/' || turf.properties.timezone == 'Antarctica/Central' || turf.properties.timezone == 'Antarctica/Mirny') { 
							antarctica++;
							insert('timezone',turf);
						} else if (turf.properties.timezone == null || turf.properties.timezone == 'Not Found') { 
							noTimeZone++;
							insert('timezone',turf);
						} else { 
							var m = (new moment(tTweet.created_at, dateFormat)).tz(turf.properties.timezone, 'YYYY-MM-DDTHH:mm:ssZ');
							turf.properties.local_timestamp = m.format(dateFormat);
							turf.properties.local_hour = m.format('HH');
							turf.properties.local_doy = m.format('DDD');
							turf.properties.local_dow = m.format('dddd');
							sum(turf.properties.country, turf.properties.point);
							insert('twitter',turf);
						}
					} else {
						countryNotFound++;
						insert('country',turf);
					}
				} else {
					noGeoCode++;
					insert('geo',tTweet);
				}
			} catch (e) {
				console.error(JSON.parse(line));
				console.error(JSON.stringify(turf));
				console.error(tTweet.created_at);
				console.error(turf.properties.timezone);
				console.error(e);
			}
		}
	}).then(function() {
		fs.unlink(fname);
		console.error('Tweets Stats:\n\tTotal Read: ' + lines + '\n\tTwitter Points: ' + summary.points + '\n\tTwitter Place: ' + summary.places + '\n\tNo Geo Coding: ' + noGeoCode + '\n\tNo Country Found: ' + countryNotFound + '\n\tNo State Found: ' + stateNotFound + '\n\tAntartica: ' + antarctica + '\n\tNo Time Zone: ' + noTimeZone);
		runLoading();
	});
}
function processFile ( fname ) { console . log ( 'Processing file: ' + fname ) ; reader . eachLine ( fname , function ( line , last ) { if ( line . substring ( line . length - 2 ) == '}}' ) { var tTweet , geojson ; lines ++ ; try { tTweet = JSON . parse ( line ) ; var turf = tweetAsGeoJson ( tTweet ) ; if ( turf ) { turf . properties . id = tTweet . id_str ; turf . properties . text = tTweet . text ; turf . properties . user_id = tTweet . user . id_str ; turf . properties . screen_name = tTweet . user . screen_name ; if ( tTweet . extended_entities && tTweet . extended_entities . media && tTweet . extended_entities . media [ 0 ] . media_url_https ) { turf . properties . media = tTweet . extended_entities . media [ 0 ] . media_url_https ; } else { turf . properties . media = "Image Not Found" ; } findState ( turf . properties . country , turf ) ; if ( turf . properties . state == 'Not Found' ) { stateNotFound ++ ; } if ( turf . properties . country == 'Not Found' ) { turf . properties . country = findCountry ( turf ) ; } if ( turf . properties . country != 'Not Found' ) { turf . properties . timezone = findTimeZone ( turf ) ; if ( turf . properties . timezone == 'Antarctica/' || turf . properties . timezone == 'Antarctica/Central' || turf . properties . timezone == 'Antarctica/Mirny' ) { antarctica ++ ; insert ( 'timezone' , turf ) ; } else if ( turf . properties . timezone == null || turf . properties . timezone == 'Not Found' ) { noTimeZone ++ ; insert ( 'timezone' , turf ) ; } else { var m = ( new moment ( tTweet . created_at , dateFormat ) ) . tz ( turf . properties . timezone , 'YYYY-MM-DDTHH:mm:ssZ' ) ; turf . properties . local_timestamp = m . format ( dateFormat ) ; turf . properties . local_hour = m . format ( 'HH' ) ; turf . properties . local_doy = m . format ( 'DDD' ) ; turf . properties . local_dow = m . format ( 'dddd' ) ; sum ( turf . properties . country , turf . properties . point ) ; insert ( 'twitter' , turf ) ; } } else { countryNotFound ++ ; insert ( 'country' , turf ) ; } } else { noGeoCode ++ ; insert ( 'geo' , tTweet ) ; } } catch ( e ) { console . error ( JSON . parse ( line ) ) ; console . error ( JSON . stringify ( turf ) ) ; console . error ( tTweet . created_at ) ; console . error ( turf . properties . timezone ) ; console . error ( e ) ; } } } ) . then ( function ( ) { fs . unlink ( fname ) ; console . error ( 'Tweets Stats:\n\tTotal Read: ' + lines + '\n\tTwitter Points: ' + summary . points + '\n\tTwitter Place: ' + summary . places + '\n\tNo Geo Coding: ' + noGeoCode + '\n\tNo Country Found: ' + countryNotFound + '\n\tNo State Found: ' + stateNotFound + '\n\tAntartica: ' + antarctica + '\n\tNo Time Zone: ' + noTimeZone ) ; runLoading ( ) ; } ) ; }
Function to read in a tweet input file and create relevant GeoJSON data
Function to read in a tweet input file and create relevant GeoJSON data
//Check each tweet //Dirty Data Check //Line Count //Line to JSON //Tweet to GeoJSON Point //Got a point? //if statement  //Find country for point //Country not found? //Find Timezone for Point //Antarctica? //In every timezone (excluded) //Null timezone? //Arctic in every timezone (excluded) //No problem //A Person without a country