My IT Journey : Social Media Analytics

Objective :

Using Open source Twitter API How to collect data from Social media - Twitter about IOT.

Twitter account :

Create a new twitter account and just follow all the IOT related groups.

So that you will get tweets from your circle and also official company groups related to IOT.

Twitter Application :

Twitter API can only connect to twitter application. Only twitter application will get the twitter message and serve to Twitter API. Our Java code get the data through twitter application .

https://apps.twitter.com/app/new is used to create a new twitter application.

Application User Credentials to connect with java application

Once we created the twitter application we can get the following user credentials.

Using the following URL you get User credentials

URL : https://apps.twitter.com/app/<applicationId>/keys

Consumer Key (API Key) : <API KEY>

Consumer Secret (API Secret) : <API Password>

Access Token : <ACCESS TOKEN>

Access Token Secret : <ACCESS TOKEN SECRET>

All these values will be generated by Twitter it self. You just need to use that values.

Note : For security purpose you can generate API key whenever you like.

Java Application - Twitter API

In Java code we are going to create a java application, which will use the generated twitter credentials . It will gather all the text from twitter related to given keyword like IOT. This java application collect all the text from twitter and store it in a text file every one hour once. So Every one hour there will be new text file which will be used to store current twitter text related to IOT.

Create a maven project and add maven dependencies

pom.xml

<groupId>com.tamil</groupId>

<artifactId>DataCollection</artifactId>

<groupId>org.twitter4j</groupId>

<artifactId>twitter4j-stream</artifactId>

</dependency>

<groupId>org.twitter4j</groupId>

<artifactId>twitter4j-core</artifactId>

</dependency>

<groupId>org.twitter4j</groupId>

<artifactId>twitter4j-async</artifactId>

</dependency>

<groupId>org.twitter4j</groupId>

<artifactId>twitter4j-media-support</artifactId>

</dependency>

</dependencies>

<build>

<artifactId>maven-assembly-plugin</artifactId>

<mainClass>com.tamil.TwitteFileWriter</mainClass>

</manifest>

</archive>

<descriptorRef>jar-with-dependencies</descriptorRef>

</descriptorRefs>

</configuration>

<id>make-assembly</id>

<phase>package</phase>

<goals>

<goal>single</goal>

</goals>

</execution>

</executions>

</plugin>

</plugins>

</build>

</project>

Maven assembly plugin to create a self executable jar file pointing to TwitteFileWriter.java

Create a java class TwitteFileWriter.java in package com.tamil as mentioned in maven pom.xml file.

TwitterFileWritter.java

package com.tamil;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileWriter;

import java.io.IOException;

import java.text.SimpleDateFormat;

import java.util.Date;

import java.util.Timer;

import java.util.TimerTask;

import twitter4j.DirectMessage;

import twitter4j.FilterQuery;

import twitter4j.StallWarning;

import twitter4j.Status;

import twitter4j.StatusDeletionNotice;

import twitter4j.TwitterStream;

import twitter4j.TwitterStreamFactory;

import twitter4j.User;

import twitter4j.UserList;

import twitter4j.UserStreamListener;

import twitter4j.conf.ConfigurationBuilder;

public class TwitteFileWriter {

private static StringBuffer buffer = new StringBuffer();

private static final UserStreamListener listener = new UserStreamListener() {

public void onException(Exception arg0) { }

public void onTrackLimitationNotice(int arg0) { }

public void onStatus(Status status) { writeTwtteFromStatus(status); }

public void onStallWarning(StallWarning arg0) { }

public void onScrubGeo(long arg0, long arg1) { }

public void onDeletionNotice(StatusDeletionNotice arg0) { }

public void onUserProfileUpdate(User arg0) { }

public void onUserListUpdate(User arg0, UserList arg1) { }

public void onUserListUnsubscription(User arg0, User arg1, UserList arg2) { }

public void onUserListSubscription(User arg0, User arg1, UserList arg2) { }

public void onUserListMemberDeletion(User arg0, User arg1, UserList arg2) { }

public void onUserListMemberAddition(User arg0, User arg1, UserList arg2) { }

public void onUserListDeletion(User arg0, UserList arg1) { }

public void onUserListCreation(User arg0, UserList arg1) { }

public void onUnfollow(User arg0, User arg1) { }

public void onUnfavorite(User arg0, User arg1,Status arg2) {}

public void onUnblock(User arg0, User arg1) { }

public void onFriendList(long[] arg0) { }

public void onFollow(User arg0, User arg1) { }

public void onFavorite(User arg0, User arg1, Status arg2) {}

public void onDirectMessage(DirectMessage arg0) { }

public void onDeletionNotice(long arg0, long arg1) { }

public void onBlock(User arg0, User arg1) { }

};

private static final void writeTwtteFromStatus(Status status) {

String tweet = status.getUser().getScreenName() + "\t"

+ status.getUser().getDescription() + "\t"

+ status.getUser().getId() + "\t"

+ status.getUser().getLang()+ "\t"

+ status.getUser().getLocation() + "\t"

+ status.getUser().getName() + "\t"

+ status.getUser().getTimeZone() + "\t"

+ status.getText()+ "\t"

+ status.getSource() + "\t"

+ status.getCreatedAt();

buffer.append("\n" + tweet);

}

public static void main(String[] args) {

Timer timer = new Timer();

RemindTask task = new RemindTask();

String[] keywords = { "Internet Of Things", "InternetOfThings","internetofthings", "IOT", "iot" };

ConfigurationBuilder conBuilder = new ConfigurationBuilder();

conBuilder.setOAuthConsumerKey(<API KEY>);

conBuilder.setOAuthConsumerSecret(<API Password>);

conBuilder.setOAuthAccessToken(<ACCESS TOKEN>);

conBuilder.setOAuthAccessTokenSecret(<ACCESS TOKEN SECRET> );

conBuilder.setJSONStoreEnabled(true);

conBuilder.setPrettyDebugEnabled(true);

// conBuilder.setHttpProxyHost("PROXY HOST");// To Set Proxy server url [optional ]

// conBuilder.setHttpProxyPort(PROXY PORT); // To Set Proxy server port [optional ]

conBuilder.setIncludeEntitiesEnabled(true);

TwitterStream twitterStream = new TwitterStreamFactory(

conBuilder.build()).getInstance();

twitterStream.addListener(listener);

twitterStream.user();

FilterQuery query = new FilterQuery().track(keywords);

twitterStream.filter(query);

timer.schedule(task, 5 * 1000, 5 * 1000);

}

static class RemindTask extends TimerTask {

private SimpleDateFormat format = new SimpleDateFormat("yyyyMMMddHH");

private boolean flagToStop = false;

private long counter = 0;

private void checkStatusToStop() {

try {

File newFile = new File("./Stop.txt");

if (newFile.exists()) {

flagToStop = true;

}

System.out.print(":" + counter);

} catch (Exception e) {

e.printStackTrace();

}

public void run() {

if (!flagToStop) {

counter++;

writerFile();

buffer.setLength(0);

checkStatusToStop();

} else {

System.out.println("Going to End");

System.exit(0);

}

private void writerFile() {

try {

String fileName = format.format(new Date());

File newFile = new File("./" + fileName + ".txt");

if (!newFile.exists())

newFile.createNewFile();

if (newFile.exists()) {

FileWriter writter = new FileWriter(newFile, true);

BufferedWriter bufferWritter = new BufferedWriter(writter);

bufferWritter.write(buffer.toString());

bufferWritter.close();

}

} catch (IOException e) {

e.printStackTrace();

}

Create Self executable jar file using maven build command

mvn clean compile install assembly:assembly

Now you can run the application using

java -jar DataCollection.jar

Application will generate txt file with the following format

<YYYY><MON><DD><HH>.txt file

Example :

2014Oct1913.txt here year 2014 , Oct month 19th at 1 pm generated file which contain twitter data collected between 1 pm to 2 am.

My IT Journey

Saturday, 25 October 2014

Social Media Analytics - Twitter API using Java

Objective :

Twitter account :

7 comments: