Application Server

Posted on April 4, 2013 by jasmeu

As mentioned in the Raspberry Pi – Surveillance WebCam in the Cloud post, the idea is to continously upload the snaphosts in the cloud in order to have them safe and accessible from outside the local network.
As there is no official Dropbox client for PI, and as I found no other solution on the web, I wrote my own little dropbox client roughly performing the following tasks:
- dropbox authentication
- continous monitoring of 2 local folders and upload to Dropbox
- deleting the oldest snaphsots before the account explodes 
The following infos were helpful for me:
- JDK 8 (with JavaFX) for ARM Early Access – Yup, you must install Java JDK on the PI, as the client is written in Java
- Josh’ Post in the Dropbox forum
- Dropbox Java SDK Tutorial

Ok, before starting, you must do several steps:
- Install Java JDK on your PI
- make sure you have an environment in which you can develop Java. Best would be outside the PI as PI’s performance might not be what you desire
- Create your own application in the Dropbox Apps page.
- Download the Dropbox Java SDK

License: Please notice the license governing the source code below in the “License & About”page.

So, here are the sources:
package eu.jasm.sync;

import java.io.File;

import com.dropbox.client2.DropboxAPI;
import com.dropbox.client2.exception.DropboxException;
import com.dropbox.client2.session.WebAuthSession;

public class SyncPics {

/**
* @param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
Authenticator auth = new Authenticator();

if (args.length > 0 && “-a”.equals(args[0])) { // we want to initiate the authentication
String url = “”;
try {
url = auth.phase1();
System.out.println(“Visit this URL: \n”+url+”\n allow the access and then restart this app without -a as parameter”);
System.out.println(“Make sure to return in the next 5 minutes; Otherwise restart the app with -a”);
System.exit(0);
} catch (DropboxException e) {
Utils.logException(e);
System.out.println(“Something wrong with your connection to DropBox; Check the stack”);
System.exit(0);
}
}
if (!Utils.propertiesExist()) {
System.out.println(“Do the authentication by calling the app with -a parameter”);
System.exit(0);
}
String phase = Utils.getProperty(Authenticator.PHASE);
//we are somehow past the initiating the authentication
if (“2″.equals(phase)) { // ok, we are in phase 2; we must get the access tokens
try {
auth.phase2();
} catch (DropboxException e) {
System.out.println(“Some error occured; Guess it in the stack below  or try to reauthenticate by calling the -a parameter”);
Utils.logException(e);
System.exit(0);
}
}
//ok, we’re here hopefully in phase 3; This means: let’s start ! 
DropboxAPI api = auth.phase3();
if (api == null) {
System.out.println(“Error occured while connecting to the Dropbox API; Try to redo the authentication by calling the app with -a parameter”);
System.exit(0);
}

int sleepTime = Utils.getSleepTime();

long remainingSize = Utils.getFreeQuota(api);
if (remainingSize == -1) {
System.out.println(“Unable to determine the gree quota for the user; Leaving …”);
System.exit(0);
}

// System.out.println(remainingSize);

long delThreshold = Utils.getDeleteThreshold();

File folder = Utils.getPath();
if (folder == null) {
System.out.println(“You must maintain the parameter “+Utils.PATH+” in the properties file “+Utils.PROPERTIES+” with the folder containing the ZIPs”);
System.exit(0);
}

if (!folder.isDirectory()) {
System.out.println(“The parameter “+Utils.PATH+” in the properties file “+Utils.PROPERTIES+” must point to a folder”);
}

File folder2 = Utils.getPath2();
if (folder2 == null) {
System.out.println(“You must maintain the parameter “+Utils.PATHBACKUP+” in the properties file “+Utils.PROPERTIES+” with the folder containing the ZIPs”);
System.exit(0);
}

if (!folder2.isDirectory()) {
System.out.println(“The parameter “+Utils.PATHBACKUP+” in the properties file “+Utils.PROPERTIES+” must point to a folder”);
}

int countFiles = 0;

while (true) {

if (countFiles > 100) { //from time to time we refresh the value
countFiles = 0;
remainingSize = Utils.getFreeQuota(api);
Utils.logDebug(“Remaining Size: “+remainingSize);
Utils.logDebug(“Delete Threshold: “+delThreshold);
}

if (remainingSize < delThreshold) { //ok we have to delete some folders
Utils.logDebug(“We must delete the oldest folder”);
Utils.deleteOldestFolder(api);
remainingSize = Utils.getFreeQuota(api);
}

File[] files = Utils.getFiles(folder);
if (files != null && files.length > 0) {
remainingSize = remainingSize – Utils.uploadFiles(files,api);
}
countFiles+=files.length;

File[] files2 = Utils.getFiles(folder2);
if (files2 != null && files2.length > 0) {
remainingSize = remainingSize – Utils.uploadFiles(files2,api);
}
countFiles+=files2.length;

try {
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
Utils.logException(e);
}
}

}

}


and:

package eu.jasm.sync;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Properties;

import com.dropbox.client2.DropboxAPI;
import com.dropbox.client2.DropboxAPI.Account;
import com.dropbox.client2.DropboxAPI.Entry;
import com.dropbox.client2.exception.DropboxException;
import com.dropbox.client2.session.WebAuthSession;

public class Utils {

public static final String PROPERTIES=”data.properties”;
private static final String SLEEPTIME = “SleepTime”;
private static final int DEFAULTSLEEPTIME = 120000;
public static final String PATH = “Path”;
public static final String PATHBACKUP = “PathBackup”;
private static final String DELTHRESHOLD = “DelThreshold”;
private static final int DEFAULTDELTHRESHOLD = 100000000;

public static String getProperty(String key) {
Properties prop = new Properties();
try {
prop.load(new FileInputStream(PROPERTIES));
return prop.getProperty(key);
} catch (IOException ex) {
Utils.logException(ex);
}
return null;
}

public static boolean propertiesExist() {
try {
@SuppressWarnings(“unused”)
FileInputStream fis = new FileInputStream(PROPERTIES);
return true;
} catch (FileNotFoundException e) {
return false;
}
}

public static void setProperty(String key, String value) { // I know, I know this is not the fastest; But: Does it REALLY matter in this case ?
Properties prop = new Properties();
try {
prop.load(new FileInputStream(PROPERTIES));
} catch (IOException ex) {
//do nothing
}
try {
prop.setProperty(key, value);
prop.store(new FileOutputStream(PROPERTIES), null);
} catch (IOException ex) {
Utils.logException(ex);
}
}

public static void logException(Exception ex) {
ex.printStackTrace();
}

public static void logInfo(String str) {
}

public static void logDebug(String str) {
System.out.println(str);
}

public static int getSleepTime() {
String timeStr = Utils.getProperty(SLEEPTIME);
if (timeStr == null || timeStr.length() == 0)
return DEFAULTSLEEPTIME;
try {
return Integer.parseInt(timeStr);
} catch (NumberFormatException ex) {
return DEFAULTSLEEPTIME;
}
}

public static File getPath() {
String pathStr = Utils.getProperty(PATH);
if (pathStr == null || pathStr.length() == 0) {
return null;
}
return new File(pathStr);
}

public static File getPath2() {
String pathStr = Utils.getProperty(PATHBACKUP);
if (pathStr == null || pathStr.length() == 0) {
return null;
}
return new File(pathStr);
}

public static File[] getFiles(File parent) {
File[] childs = parent.listFiles();
Arrays.sort(childs);
return childs;
}

public static long uploadFiles(File[] files,DropboxAPI api) {
java.io.FileInputStream inputStream = null;
Entry newEntry;
long uploadedSize = 0;
Date dateNow = new Date();
String dateStr = new SimpleDateFormat(“yyyyMMdd”).format(dateNow);
String hourStr = new SimpleDateFormat(“HH”).format(dateNow);

for (int i=0;i<files.length;i++) {
if (files[i] == null || files[i].isDirectory()) continue;
if (!files[i].getName().endsWith(“.zip”) && !files[i].getName().endsWith(“.jpeg”)) continue;
try {
inputStream = new java.io.FileInputStream(files[i]);
if (files[i].getName().endsWith(“.zip”)) {
newEntry = api.putFile(“/zip/”+files[i].getName(), inputStream, files[i].length(), null, null);
} else {
newEntry = api.putFile(“/”+dateStr+”/”+hourStr+”/”+files[i].getName(), inputStream, files[i].length(), null, null);
}
uploadedSize+=newEntry.bytes;
files[i].delete();
} catch(Exception e) {
System.out.println(“Some error occured; Guess it in the stack below  ”);
Utils.logException(e);
}
finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException ex) {
Utils.logException(ex);
}
}
}
}
return uploadedSize;
}

public static long getFreeQuota(DropboxAPI api) {
try {
Account accountInfo = api.accountInfo();
return accountInfo.quota – accountInfo.quotaNormal – accountInfo.quotaShared;
} catch (DropboxException e) {
Utils.logException(e);
}
return -1;
}

public static long getDeleteThreshold() {
String threshold = Utils.getProperty(DELTHRESHOLD);
if (threshold == null || threshold.length() == 0)
return DEFAULTDELTHRESHOLD;
try {
return Long.parseLong(threshold);
} catch (NumberFormatException ex) {
return DEFAULTSLEEPTIME;
}
}

public static void deleteOldestFolder(DropboxAPI api) {
try {
Entry metadata = api.metadata(“/”, -1, null, true, null);
List childs = metadata.contents;
if (childs == null || childs.size() == 0) return;

Date dateNow = new Date();
String dateStr = new SimpleDateFormat(“yyyyMMdd”).format(dateNow);

ArrayList dirNames = new ArrayList();

for (int i=0;i < childs.size();i++) {
Entry child = childs.get(i);
if (child != null && !child.isDeleted && child.isDir) {
String fileName = child.fileName();
if (!dateStr.equals(fileName))
dirNames.add(fileName);
}
}

if (dirNames.size() == 0) return;

java.util.Collections.sort(dirNames);
String oldestFolder = dirNames.get(0);

Utils.logDebug(“Oldest folder to be deleted is:”+oldestFolder);
//as we have too many files, we must delete the children first
metadata = api.metadata(“/”+oldestFolder+”/”, -1, null, true, null);
childs = metadata.contents;
for (int i=0;i < childs.size();i++) {
Entry child = childs.get(i);
if (child != null && !child.isDeleted && child.isDir) {
String fileName = child.fileName();
Utils.logDebug(“Children: /”+oldestFolder+”/”+fileName+” to be deleted”);
api.delete(“/”+oldestFolder+”/”+fileName);
}
}
api.delete(“/”+oldestFolder);

} catch (DropboxException e) {
System.out.println(“Some error occured; Guess it in the stack below  ”);
Utils.logException(e);
return;
}
}

}


and last but not least:

package eu.jasm.sync;

import com.dropbox.client2.DropboxAPI;
import com.dropbox.client2.exception.DropboxException;
import com.dropbox.client2.session.AccessTokenPair;
import com.dropbox.client2.session.AppKeyPair;
import com.dropbox.client2.session.RequestTokenPair;
import com.dropbox.client2.session.WebAuthSession;
import com.dropbox.client2.session.Session.AccessType;
import com.dropbox.client2.session.WebAuthSession.WebAuthInfo;

public class Authenticator {

private static final String APPKEY = “Enter here you app key“;
private static final String APPSECRET = “Enter here your app secret“;

private static final String REQKEY = “RequestKey”;
private static final String REQSECRET = “RequestSecret”;

private static final String AccessKEY = “AccessKey”;
private static final String AccessSECRET = “AccessSecret”;

public static final String PHASE = “Phase”;

private static final AccessType ACCESS_TYPE = AccessType.APP_FOLDER;

public String phase1() throws DropboxException {

AppKeyPair appKeys = new AppKeyPair(APPKEY, APPSECRET);
WebAuthSession session = new WebAuthSession(appKeys, ACCESS_TYPE);
WebAuthInfo authInfo = session.getAuthInfo();

RequestTokenPair requestToken = authInfo.requestTokenPair;
Utils.setProperty(REQKEY, requestToken.key);
Utils.setProperty(REQSECRET, requestToken.secret);
Utils.setProperty(PHASE, “2″);
return authInfo.url;
}

public void phase2() throws DropboxException {
RequestTokenPair requestToken = new RequestTokenPair(Utils.getProperty(REQKEY), Utils.getProperty(REQSECRET));
AppKeyPair appKeys = new AppKeyPair(APPKEY, APPSECRET);
WebAuthSession session = new WebAuthSession(appKeys, ACCESS_TYPE);
session.retrieveWebAccessToken(requestToken);
AccessTokenPair accessToken = session.getAccessTokenPair();
Utils.setProperty(AccessKEY, accessToken.key);
Utils.setProperty(AccessSECRET, accessToken.secret);
Utils.setProperty(REQKEY, “”);
Utils.setProperty(REQSECRET, “”);
Utils.setProperty(PHASE, “3″);
}

public DropboxAPI phase3() {
AppKeyPair appKeys = new AppKeyPair(APPKEY, APPSECRET);
WebAuthSession session = new WebAuthSession(appKeys, ACCESS_TYPE);
String accKey,accSecret;
accKey = Utils.getProperty(AccessKEY);
accSecret = Utils.getProperty(AccessSECRET);
if (accKey == null || accSecret == null || accKey.length() == 0 || accSecret.length() == 0) {
return null;
}
AccessTokenPair accessToken = new AccessTokenPair(accKey, accSecret);
session.setAccessTokenPair(accessToken);
DropboxAPI api = new DropboxAPI(session);
return api;
}
}


This code tries to read from a data.properties file whose content you may initially set like this:

Phase=1
SleepTime=80000
Path=/home/pi/Shared/ram/trgt
PathBackup=/home/pi/Shared/backup
DelThreshold=1000000000

I think the params should be clear, especially by considering the info from the first post (Raspberry Pi – Surveillance WebCam in the Cloud). Maybe only DelThreshold is new: When the empty space left in the dropbox account is no bigger than DelThreshold, the client starts to delete the oldest content uploaded by this client.

Finally, as always, for comments and discussion, let’s use the thread @ Raspberry PI Forum.
And btw, in case you’ll create yourself a dropbox account as result of this post, please feel free to use this link. I might use well some free MBs for the application above