Wednesday, September 29, 2010

A simple versioning implementation of Voldemort

I've been playing around with Voldemort recently for a robust enormous datastore at Esped.com.  One of the frustrating things I found in getting into it was the lack of complete existing code samples, so I thought I'd contribute one to the community.

This is a bit more than simple put/get because I need versioning associated with my objects.  This code assumes you've got a Voldemort server up and running.  Make the object you want to save implement Storeable, and then you can call VoldemortValet.process(yourObject) and it will save it.

It would be easy enough to take the versioning OUT of this code if you needed to.






import org.jdom.Element;

public interface Storeable {
    public String getIidString();
    public Element rawXML();
    public String getUserId();
    public String getDataType();
}



/**
 * A simple versioning datastore.  In this example we want to keep track of every iteration of every object
 * as well as every change made by every user.  In a relationalDB we'd have a userid in the versions of the objects, but
 * noSQL doesn't work that way.  We use the ID of the object as a storage point in the store for the current version of
 * the object we're storing.  When we store a new version of an object, we increment the version in Voldermort, and the
 * store the object with <id>_<version> as the key.
 * <p/>
 * Thus if you want to retrieve all the entries for an object with id='3141529' you'd first query Voldemort to see how many
 * versions there were, and then get every version by specific key
 */
public class VoldemortValet {

    public static SimpleDateFormat datetime = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    static StoreClientFactory factory;
    static String bootstrapUrl = "tcp://192.168.0.231:6666";
    static String storeName = "myStore";

    static {
        ClientConfig cc = new ClientConfig();
        cc.setBootstrapUrls(bootstrapUrl);
        factory = new SocketStoreClientFactory(cc);
    }

    public static void main(String argv[]) {
        DefaultStoreClient<Object, Object> client = null;
        try {
            client = (DefaultStoreClient<Object, Object>) factory.getStoreClient(storeName);
        } catch (Exception e) {
            System.out.println("Could not connect to server: " + e.getMessage());
        }
        client.put("hello", "new world:" + new Date().toString());
    }

    /**
     * When saving a Storeable, we first find out the version of the we will be saving, and then save takt with a key
     * of IID_V where IID is the id of the object and V is the version number we want to save with.
     *
     * @param
     */
    public static boolean process(Storeable so) {
        DefaultStoreClient<Object, Object> client = null;
        //create the XML we'll be saving and add the time and updating user id to it
        Element element = so.rawXML();
        element.setAttribute("CALLED_BY", so.getUserId());
        element.setAttribute("IVDATE", datetime.format(new Date()));
        //convert the JDOM element to a string
        String uxml = new XMLOutputter().outputString(element);
        try {
            //connect to the client
            client = (DefaultStoreClient<Object, Object>) factory.getStoreClient(storeName);
        } catch (Exception e) {
            System.out.println("Could not connect to server: " + e.getMessage());
            return false;
        }
        //get the version number counter, defaulting to 0 if there is nothing in Voldemort
        String versionCounter = (String) client.getValue(so.getIidString(), "0");
        //get the update number counter this user has done, by querying with the userid
        String updateCounter = (String) client.getValue(so.getUserId(), "0");
        //iterate the counters
        versionCounter = Integer.toString(Integer.parseInt(versionCounter) + 1);
        updateCounter = Integer.toString(Integer.parseInt(updateCounter) + 1);
        //store the updated counters
        client.put(so.getIidString(), versionCounter);
        client.put(so.getUserId(), updateCounter);
        //store the newest version of the Storeable
        client.put(so.getIidString() + "_" + versionCounter, uxml);
        //store the transaction with the user's ID, so we could find all the updates he/she did if we wanted to
        client.put(so.getUserId() + "_" + updateCounter, so.getDataType() + "_" + so.getIidString() + "_" + versionCounter);
        return true;
    }

    /**
     * This gets back a vector of all the last N version of an object
     *
     * @param key
     * @param numberOfVersions
     * @return
     */
    public static Vector get(String key, int numberOfVersions) {
        DefaultStoreClient<Object, Object> client = null;
        Vector v = new Vector();
        try {
            client = (DefaultStoreClient<Object, Object>) factory.getStoreClient(storeName);
            //client = (DefaultStoreClient<Object, Object>) factory.getStoreClient(String.valueOf(eo.getOrganizationID()));
        } catch (Exception e) {
            System.out.println("Could not connect to server: " + e.getMessage());
            return null;
        }
        //how many versions of this object are there?
        String version = (String) client.getValue(key, "0");
        int versionNumbers = Integer.parseInt(version);
        //prepare to make a JDOM element out of the stored XML
        SAXBuilder builder = new SAXBuilder();
        Document xdoc = null;
        //get the specific versions
        for (int x = 0; versionNumbers - x > 0 && x < numberOfVersions; x++) {
            String rxml = (String) client.getValue(key + "_" + Integer.toString(versionNumbers - x));
            try {
                //note that this makes a document and puts the stored element in as the root element.
                xdoc = builder.build(new StringReader(rxml));
                v.add(xdoc);
            } catch (JDOMException e) {
                e.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return v;
    }


} 




/* code syntax highlights courtesy of http://www.manoli.net/csharpformat/ I used the C# option, and it worked well for java. */

No comments :