Implementation of MerkleDag get and put

Now saving and retrieving MerkleDags that contain data. Now need to work
with links and other types.

include/ipfs/merkledag/merkledag.h

@@ -12,5 +12,13 @@
  */
 int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo);
 
+/***
+ * Retrieves a node from the datastore based on the cid
+ * @param cid the key to look for
+ * @param node the node to be created
+ * @param fs_repo the repository
+ * @returns true(1) on success
+ */
+int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo);
 
 #endif

include/ipfs/repo/config/config.h

@@ -72,7 +72,7 @@ int config_path(char* config_root, char* extension, char* result, int max_len);
  * @param num_bits_for_keypair number of bits for the key pair
  * @returns true(1) on success, otherwise 0
  */
-int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path);
+int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, const char* repo_path);
 
 /***
  * Initialize memory for a RepoConfig struct

include/ipfs/repo/config/datastore.h

@@ -20,8 +20,10 @@ struct Datastore {
 	// function pointers for datastore operations
 	int (*datastore_open)(int argc, char** argv, struct Datastore* datastore);
 	int (*datastore_close)(struct Datastore* datastore);
-	int (*datastore_put)(const char* key, size_t key_size, unsigned char* data, size_t data_length, struct Datastore* datastore);
+	int (*datastore_put)(const unsigned char* key, size_t key_size, unsigned char* data, size_t data_length, const struct Datastore* datastore);
-	//int (*datastore_get)(const char* key, struct Block* block);
+	int (*datastore_put_block)(const struct Block* block, const struct Datastore* datastore);
+	int (*datastore_get)(const char* key, size_t key_size, unsigned char* data, size_t max_data_length, size_t* data_length, const struct Datastore* datastore);
+	int (*datastore_get_block)(const struct Cid* cid, struct Block** block, const struct Datastore* datastore);
 	// a handle to the datastore "context" used by the datastore
 	void* handle;
 };
@@ -33,7 +35,7 @@ struct Datastore {
  * @param config_root the path to the root of IPFS
  * @returns true(1) on success
  */
-int ipfs_repo_config_datastore_init(struct Datastore* datastore, char* config_root);
+int ipfs_repo_config_datastore_init(struct Datastore* datastore, const char* config_root);
 
 /***
  * initialize the structure of the datastore

include/ipfs/repo/fsrepo/fs_repo.h

@@ -48,7 +48,7 @@ int fs_repo_write_config_file(char* path, struct RepoConfig* config);
  * @param repo the struct to allocate memory for
  * @returns false(0) if something bad happened, otherwise true(1)
  */
-int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** fs_repo);
+int ipfs_repo_fsrepo_new(const char* repo_path, struct RepoConfig* config, struct FSRepo** fs_repo);
 
 /***
  * Free all resources used by this struct

merkledag/merkledag.c

@@ -17,7 +17,7 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
 	struct Block* block;
 	ipfs_blocks_block_new(node->data, node->data_size, &block);
 
-	int retVal = fs_repo->config->datastore->datastore_put(block->cid->hash, block->cid->hash_length, block->data, block->data_length, fs_repo->config->datastore);
+	int retVal = fs_repo->config->datastore->datastore_put_block(block, fs_repo->config->datastore);
 	if (retVal == 0) {
 		ipfs_blocks_block_free(block);
 		return 0;
@@ -29,3 +29,26 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
 	// TODO: call HasBlock (unsure why as yet)
 	return 1;
 }
+
+/***
+ * Retrieves a node from the datastore based on the cid
+ * @param cid the key to look for
+ * @param node the node to be created
+ * @param fs_repo the repository
+ * @returns true(1) on success
+ */
+int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo) {
+	int retVal = 1;
+
+	// look for the block
+	struct Block* block;
+	retVal = fs_repo->config->datastore->datastore_get_block(cid, &block, fs_repo->config->datastore);
+	if (retVal == 0)
+		return 0;
+
+	// we have the block. Fill the node
+	*node = N_Create_From_Data(block->data, block->data_length);
+	Node_Set_Cid(*node, cid);
+
+	return retVal;
+}

repo/config/config.c

@@ -79,7 +79,7 @@ int repo_config_get_file_name(char* path, char** result) {
  * @param num_bits_for_keypair number of bits for the key pair
  * @returns true(1) on success, otherwise 0
  */
-int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path) {
+int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, const char* repo_path) {
 	// identity
 	int retVal = repo_config_identity_init(config->identity, num_bits_for_keypair);
 	if (retVal == 0)

repo/config/datastore.c

@@ -17,7 +17,7 @@ int alloc_and_assign(char** result, const char* string) {
  * @param datastore the struct to initialize
  * @returns true(1) on success
  */
-int ipfs_repo_config_datastore_init(struct Datastore* datastore, char* config_root) {
+int ipfs_repo_config_datastore_init(struct Datastore* datastore, const char* config_root) {
 	unsigned long stringLength = strlen(config_root) + 12;
 	datastore->path = malloc(sizeof(char) * stringLength);
 	os_utils_filepath_join(config_root, "datastore", datastore->path, stringLength);

repo/fsrepo/fs_repo.c

@@ -108,7 +108,7 @@ int repo_config_write_config_file(char* full_filename, struct RepoConfig* config
  * @param repo the struct to allocate memory for
  * @returns false(0) if something bad happened, otherwise true(1)
  */
-int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** repo) {
+int ipfs_repo_fsrepo_new(const char* repo_path, struct RepoConfig* config, struct FSRepo** repo) {
 	*repo = (struct FSRepo*)malloc(sizeof(struct FSRepo));
 
 	if (repo_path == NULL) {

repo/fsrepo/lmdb_datastore.c

@@ -10,13 +10,123 @@
 #include "lmdb.h"
 #include "ipfs/repo/fsrepo/lmdb_datastore.h"
 
-/**
+int repo_fsrepo_lmdb_get_block(const struct Cid* cid, struct Block** block, const struct Datastore* datastore) {
- * Write a block to the datastore with the specified key
+	int retVal;
- * @param key the key
+	MDB_txn* mdb_txn;
- * @param block the block to be written
+	MDB_dbi mdb_dbi;
+	struct MDB_val db_key;
+	struct MDB_val db_value;
+
+	MDB_env* mdb_env = (MDB_env*)datastore->handle;
+	if (mdb_env == NULL)
+		return 0;
+
+	// open transaction
+	retVal = mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn);
+	if (retVal != 0)
+		return 0;
+	retVal = mdb_dbi_open(mdb_txn, NULL, MDB_DUPSORT, &mdb_dbi);
+	if (retVal != 0) {
+		mdb_txn_commit(mdb_txn);
+		return 0;
+	}
+
+	// prepare data
+	db_key.mv_size = cid->hash_length;
+	db_key.mv_data = cid->hash;
+
+	retVal = mdb_get(mdb_txn, mdb_dbi, &db_key, &db_value);
+	if (retVal != 0) {
+		mdb_dbi_close(mdb_env, mdb_dbi);
+		mdb_txn_commit(mdb_txn);
+		return 0;
+	}
+
+	// now copy the data
+	retVal = ipfs_blocks_block_new(db_value.mv_data, db_value.mv_size, block);
+	if (retVal == 0) {
+		mdb_dbi_close(mdb_env, mdb_dbi);
+		mdb_txn_commit(mdb_txn);
+		return 0;
+	}
+
+	// clean up
+	mdb_dbi_close(mdb_env, mdb_dbi);
+	mdb_txn_commit(mdb_txn);
+
+	return 1;
+}
+
+/***
+ * retrieve a record from the database and put in a pre-sized buffer
+ * @param key the key to look for
+ * @param key_size the length of the key
+ * @param data the data that is retrieved
+ * @param max_data_size the length of the data buffer
+ * @param data_size the length of the data that was found in the database
+ * @param datastore where to look for the data
  * @returns true(1) on success
  */
-int repo_fsrepo_lmdb_put(const char* key, size_t key_size, unsigned char* data, size_t data_size, struct Datastore* datastore) {
+int repo_fsrepo_lmdb_get(const char* key, size_t key_size, unsigned char* data, size_t max_data_size, size_t* data_size, const struct Datastore* datastore) {
+	int retVal;
+	MDB_txn* mdb_txn;
+	MDB_dbi mdb_dbi;
+	struct MDB_val db_key;
+	struct MDB_val db_value;
+
+	MDB_env* mdb_env = (MDB_env*)datastore->handle;
+	if (mdb_env == NULL)
+		return 0;
+
+	// open transaction
+	retVal = mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn);
+	if (retVal != 0)
+		return 0;
+	retVal = mdb_dbi_open(mdb_txn, NULL, MDB_DUPSORT, &mdb_dbi);
+	if (retVal != 0) {
+		mdb_txn_commit(mdb_txn);
+		return 0;
+	}
+
+	// prepare data
+	db_key.mv_size = key_size;
+	db_key.mv_data = (char*)key;
+
+	retVal = mdb_get(mdb_txn, mdb_dbi, &db_key, &db_value);
+	if (retVal != 0) {
+		mdb_dbi_close(mdb_env, mdb_dbi);
+		mdb_txn_commit(mdb_txn);
+		return 0;
+	}
+
+	// now copy the data
+	if (db_value.mv_size > max_data_size) {
+		mdb_dbi_close(mdb_env, mdb_dbi);
+		mdb_txn_commit(mdb_txn);
+		return 0;
+	}
+
+	// set return values
+	memcpy(data, db_value.mv_data, db_value.mv_size);
+	(*data_size) = db_value.mv_size;
+
+	// clean up
+	mdb_dbi_close(mdb_env, mdb_dbi);
+	mdb_txn_commit(mdb_txn);
+
+	return 1;
+}
+
+/**
+ * Write data to the datastore with the specified key
+ * @param key the key
+ * @param key_size the length of the key
+ * @param data the data to be written
+ * @param data_size the length of the data to be written
+ * @param datastore the datastore to write to
+ * @returns true(1) on success
+ */
+int repo_fsrepo_lmdb_put(unsigned const char* key, size_t key_size, unsigned char* data, size_t data_size, const struct Datastore* datastore) {
 	int retVal;
 	MDB_txn* mdb_txn;
 	MDB_dbi mdb_dbi;
@@ -58,7 +168,16 @@ int repo_fsrepo_lmdb_put(const char* key, size_t key_size, unsigned char* data,
 	return retVal;
 }
 
+/**
+ * Write a block to the datastore with the specified key
+ * @param block the block to be written
+ * @param datastore the datastore to write to
+ * @returns true(1) on success
+ */
+int repo_fsrepo_lmdb_put_block(const struct Block* block, const struct Datastore* datastore) {
 
+	return repo_fsrepo_lmdb_put(block->cid->hash, block->cid->hash_length, block->data, block->data_length, datastore);
+}
 
 /**
  * Open an lmdb database with the given parameters.
@@ -108,7 +227,9 @@ int repo_fsrepo_lmdb_cast(struct Datastore* datastore) {
 	datastore->datastore_open = &repo_fsrepro_lmdb_open;
 	datastore->datastore_close = &repo_fsrepo_lmdb_close;
 	datastore->datastore_put = &repo_fsrepo_lmdb_put;
-	//datastore->datastore_get = &repo_fsrepo_lmdb_get;
+	datastore->datastore_put_block = &repo_fsrepo_lmdb_put_block;
+	datastore->datastore_get = &repo_fsrepo_lmdb_get;
+	datastore->datastore_get_block = &repo_fsrepo_lmdb_get_block;
 	return 1;
 }
 

test/merkledag/test_merkledag.h

@@ -2,6 +2,68 @@
 #include "ipfs/node/node.h"
 #include "../test_helper.h"
 
+int test_merkledag_get_data() {
+	int retVal = 0;
+
+	// create a fresh repo
+	retVal = drop_and_build_repository("/tmp/.ipfs");
+	if (retVal == 0)
+		return 0;
+
+	// open the fs repo
+	struct RepoConfig* repo_config = NULL;
+	struct FSRepo* fs_repo;
+	const char* path = "/tmp/.ipfs";
+
+	// create the struct
+	retVal = ipfs_repo_fsrepo_new((char*)path, repo_config, &fs_repo);
+	if (retVal == 0)
+		return 0;
+
+	// open the repository and read the config file
+	retVal = ipfs_repo_fsrepo_open(fs_repo);
+	if (retVal == 0) {
+		ipfs_repo_fsrepo_free(fs_repo);
+		return 0;
+	}
+
+	// get the size of the database
+	int start_file_size = os_utils_file_size("/tmp/.ipfs/datastore/data.mdb");
+
+	// create data for node
+	size_t binary_data_size = 256;
+	unsigned char binary_data[binary_data_size];
+	for(int i = 0; i < binary_data_size; i++) {
+		binary_data[i] = i;
+	}
+
+	// create a node
+	struct Node* node1 = N_Create_From_Data(binary_data, 256);
+
+	retVal = ipfs_merkledag_add(node1, fs_repo);
+	if (retVal == 0) {
+		Node_Delete(node1);
+		return 0;
+	}
+
+	// now retrieve it
+	struct Node* results_node;
+	retVal = ipfs_merkledag_get(node1->cached, &results_node, fs_repo);
+	if (retVal == 0)
+		return 0;
+
+	if (results_node->data_size != 256)
+		return 0;
+
+	// the data should be the same
+	for(int i = 0; i < results_node->data_size; i++) {
+		if (results_node->data[i] != node1->data[i])
+			return 0;
+	}
+
+	return retVal;
+}
+
 int test_merkledag_add_data() {
 	int retVal = 0;
 
@@ -31,14 +93,14 @@ int test_merkledag_add_data() {
 	int start_file_size = os_utils_file_size("/tmp/.ipfs/datastore/data.mdb");
 
 	// create data for node
-	size_t binary_data_size = 255;
+	size_t binary_data_size = 256;
 	unsigned char binary_data[binary_data_size];
 	for(int i = 0; i < binary_data_size; i++) {
 		binary_data[i] = i;
 	}
 
 	// create a node
-	struct Node* node1 = N_Create_From_Data(binary_data, 255);
+	struct Node* node1 = N_Create_From_Data(binary_data, 256);
 
 	retVal = ipfs_merkledag_add(node1, fs_repo);
 	if (retVal == 0) {

test/storage/test_blocks.h

@@ -1,18 +1,18 @@
 #include "ipfs/blocks/block.h"
 
 int test_blocks_new() {
-	const char* input = "Hello, World!";
+	const unsigned char* input = (const unsigned char*)"Hello, World!";
 	int retVal = 0;
 	struct Block* block;
-	retVal = ipfs_blocks_block_new(input, strlen(input) + 1, &block);
+	retVal = ipfs_blocks_block_new(input, strlen((const char*)input) + 1, &block);
 	if (retVal == 0)
 		return 0;
 
 	// now examine the block
-	if (strcmp(block->data, input) != 0)
+	if (strcmp((const char*)block->data, (const char*)input) != 0)
 		return 0;
 
-	if (block->data_length != strlen(input) + 1)
+	if (block->data_length != strlen((const char*)input) + 1)
 		return 0;
 
 	if (block->cid->codec != CID_PROTOBUF)

test/storage/test_datastore.h

@@ -14,8 +14,8 @@
  */
 int test_ipfs_datastore_put() {
 	struct Block* block;
-	int retVal;
+	int retVal = 0;
-	const unsigned char* input = "Hello, world!";
+	const unsigned char* input = (unsigned char*)"Hello, world!";
 
 	// build the ipfs repository, then shut it down, so we can start fresh
 	drop_and_build_repository("/tmp/.ipfs");
@@ -44,7 +44,7 @@ int test_ipfs_datastore_put() {
 		return 0;
 
 	// send to Put with key
-	retVal = fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
+	retVal = fs_repo->config->datastore->datastore_put((const unsigned char*)key, key_length, block->data, block->data_length, fs_repo->config->datastore);
 	if (retVal == 0)
 		return 0;
 

test/test_helper.c

@@ -1,7 +1,9 @@
 #include <dirent.h>
 #include <sys/stat.h>
+#include <string.h>
 
 #include "ipfs/repo/fsrepo/fs_repo.h"
+#include "ipfs/os/utils.h"
 
 int remove_directory(const char *path)
 {

test/testit.c

@@ -41,7 +41,8 @@ const char* names[] = {
 		"test_repo_bootstrap_peers_init",
 		"test_ipfs_datastore_put",
 		"test_node",
-		"test_merkledag_add_data"
+		"test_merkledag_add_data",
+		"test_merkledag_get_data"
 };
 
 int (*funcs[])(void) = {
@@ -64,7 +65,8 @@ int (*funcs[])(void) = {
 		test_repo_bootstrap_peers_init,
 		test_ipfs_datastore_put,
 		test_node,
-		test_merkledag_add_data
+		test_merkledag_add_data,
+		test_merkledag_get_data
 };
 
 /**