Implementation of MerkleDag get and put

Now saving and retrieving MerkleDags that contain data. Now need to work
with links and other types.
This commit is contained in:
jmjatlanta 2016-12-05 17:23:58 -05:00
parent 8a80d2afc7
commit da6490ac7f
14 changed files with 246 additions and 26 deletions

View file

@ -12,5 +12,13 @@
*/ */
int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo); int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo);
/***
* Retrieves a node from the datastore based on the cid
* @param cid the key to look for
* @param node the node to be created
* @param fs_repo the repository
* @returns true(1) on success
*/
int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo);
#endif #endif

View file

@ -72,7 +72,7 @@ int config_path(char* config_root, char* extension, char* result, int max_len);
* @param num_bits_for_keypair number of bits for the key pair * @param num_bits_for_keypair number of bits for the key pair
* @returns true(1) on success, otherwise 0 * @returns true(1) on success, otherwise 0
*/ */
int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path); int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, const char* repo_path);
/*** /***
* Initialize memory for a RepoConfig struct * Initialize memory for a RepoConfig struct

View file

@ -20,8 +20,10 @@ struct Datastore {
// function pointers for datastore operations // function pointers for datastore operations
int (*datastore_open)(int argc, char** argv, struct Datastore* datastore); int (*datastore_open)(int argc, char** argv, struct Datastore* datastore);
int (*datastore_close)(struct Datastore* datastore); int (*datastore_close)(struct Datastore* datastore);
int (*datastore_put)(const char* key, size_t key_size, unsigned char* data, size_t data_length, struct Datastore* datastore); int (*datastore_put)(const unsigned char* key, size_t key_size, unsigned char* data, size_t data_length, const struct Datastore* datastore);
//int (*datastore_get)(const char* key, struct Block* block); int (*datastore_put_block)(const struct Block* block, const struct Datastore* datastore);
int (*datastore_get)(const char* key, size_t key_size, unsigned char* data, size_t max_data_length, size_t* data_length, const struct Datastore* datastore);
int (*datastore_get_block)(const struct Cid* cid, struct Block** block, const struct Datastore* datastore);
// a handle to the datastore "context" used by the datastore // a handle to the datastore "context" used by the datastore
void* handle; void* handle;
}; };
@ -33,7 +35,7 @@ struct Datastore {
* @param config_root the path to the root of IPFS * @param config_root the path to the root of IPFS
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_repo_config_datastore_init(struct Datastore* datastore, char* config_root); int ipfs_repo_config_datastore_init(struct Datastore* datastore, const char* config_root);
/*** /***
* initialize the structure of the datastore * initialize the structure of the datastore

View file

@ -48,7 +48,7 @@ int fs_repo_write_config_file(char* path, struct RepoConfig* config);
* @param repo the struct to allocate memory for * @param repo the struct to allocate memory for
* @returns false(0) if something bad happened, otherwise true(1) * @returns false(0) if something bad happened, otherwise true(1)
*/ */
int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** fs_repo); int ipfs_repo_fsrepo_new(const char* repo_path, struct RepoConfig* config, struct FSRepo** fs_repo);
/*** /***
* Free all resources used by this struct * Free all resources used by this struct

View file

@ -17,7 +17,7 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
struct Block* block; struct Block* block;
ipfs_blocks_block_new(node->data, node->data_size, &block); ipfs_blocks_block_new(node->data, node->data_size, &block);
int retVal = fs_repo->config->datastore->datastore_put(block->cid->hash, block->cid->hash_length, block->data, block->data_length, fs_repo->config->datastore); int retVal = fs_repo->config->datastore->datastore_put_block(block, fs_repo->config->datastore);
if (retVal == 0) { if (retVal == 0) {
ipfs_blocks_block_free(block); ipfs_blocks_block_free(block);
return 0; return 0;
@ -29,3 +29,26 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
// TODO: call HasBlock (unsure why as yet) // TODO: call HasBlock (unsure why as yet)
return 1; return 1;
} }
/***
* Retrieves a node from the datastore based on the cid
* @param cid the key to look for
* @param node the node to be created
* @param fs_repo the repository
* @returns true(1) on success
*/
int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo) {
int retVal = 1;
// look for the block
struct Block* block;
retVal = fs_repo->config->datastore->datastore_get_block(cid, &block, fs_repo->config->datastore);
if (retVal == 0)
return 0;
// we have the block. Fill the node
*node = N_Create_From_Data(block->data, block->data_length);
Node_Set_Cid(*node, cid);
return retVal;
}

View file

@ -79,7 +79,7 @@ int repo_config_get_file_name(char* path, char** result) {
* @param num_bits_for_keypair number of bits for the key pair * @param num_bits_for_keypair number of bits for the key pair
* @returns true(1) on success, otherwise 0 * @returns true(1) on success, otherwise 0
*/ */
int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path) { int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, const char* repo_path) {
// identity // identity
int retVal = repo_config_identity_init(config->identity, num_bits_for_keypair); int retVal = repo_config_identity_init(config->identity, num_bits_for_keypair);
if (retVal == 0) if (retVal == 0)

View file

@ -17,7 +17,7 @@ int alloc_and_assign(char** result, const char* string) {
* @param datastore the struct to initialize * @param datastore the struct to initialize
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_repo_config_datastore_init(struct Datastore* datastore, char* config_root) { int ipfs_repo_config_datastore_init(struct Datastore* datastore, const char* config_root) {
unsigned long stringLength = strlen(config_root) + 12; unsigned long stringLength = strlen(config_root) + 12;
datastore->path = malloc(sizeof(char) * stringLength); datastore->path = malloc(sizeof(char) * stringLength);
os_utils_filepath_join(config_root, "datastore", datastore->path, stringLength); os_utils_filepath_join(config_root, "datastore", datastore->path, stringLength);

View file

@ -108,7 +108,7 @@ int repo_config_write_config_file(char* full_filename, struct RepoConfig* config
* @param repo the struct to allocate memory for * @param repo the struct to allocate memory for
* @returns false(0) if something bad happened, otherwise true(1) * @returns false(0) if something bad happened, otherwise true(1)
*/ */
int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** repo) { int ipfs_repo_fsrepo_new(const char* repo_path, struct RepoConfig* config, struct FSRepo** repo) {
*repo = (struct FSRepo*)malloc(sizeof(struct FSRepo)); *repo = (struct FSRepo*)malloc(sizeof(struct FSRepo));
if (repo_path == NULL) { if (repo_path == NULL) {

View file

@ -10,13 +10,123 @@
#include "lmdb.h" #include "lmdb.h"
#include "ipfs/repo/fsrepo/lmdb_datastore.h" #include "ipfs/repo/fsrepo/lmdb_datastore.h"
/** int repo_fsrepo_lmdb_get_block(const struct Cid* cid, struct Block** block, const struct Datastore* datastore) {
* Write a block to the datastore with the specified key int retVal;
* @param key the key MDB_txn* mdb_txn;
* @param block the block to be written MDB_dbi mdb_dbi;
struct MDB_val db_key;
struct MDB_val db_value;
MDB_env* mdb_env = (MDB_env*)datastore->handle;
if (mdb_env == NULL)
return 0;
// open transaction
retVal = mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn);
if (retVal != 0)
return 0;
retVal = mdb_dbi_open(mdb_txn, NULL, MDB_DUPSORT, &mdb_dbi);
if (retVal != 0) {
mdb_txn_commit(mdb_txn);
return 0;
}
// prepare data
db_key.mv_size = cid->hash_length;
db_key.mv_data = cid->hash;
retVal = mdb_get(mdb_txn, mdb_dbi, &db_key, &db_value);
if (retVal != 0) {
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 0;
}
// now copy the data
retVal = ipfs_blocks_block_new(db_value.mv_data, db_value.mv_size, block);
if (retVal == 0) {
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 0;
}
// clean up
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 1;
}
/***
* retrieve a record from the database and put in a pre-sized buffer
* @param key the key to look for
* @param key_size the length of the key
* @param data the data that is retrieved
* @param max_data_size the length of the data buffer
* @param data_size the length of the data that was found in the database
* @param datastore where to look for the data
* @returns true(1) on success * @returns true(1) on success
*/ */
int repo_fsrepo_lmdb_put(const char* key, size_t key_size, unsigned char* data, size_t data_size, struct Datastore* datastore) { int repo_fsrepo_lmdb_get(const char* key, size_t key_size, unsigned char* data, size_t max_data_size, size_t* data_size, const struct Datastore* datastore) {
int retVal;
MDB_txn* mdb_txn;
MDB_dbi mdb_dbi;
struct MDB_val db_key;
struct MDB_val db_value;
MDB_env* mdb_env = (MDB_env*)datastore->handle;
if (mdb_env == NULL)
return 0;
// open transaction
retVal = mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn);
if (retVal != 0)
return 0;
retVal = mdb_dbi_open(mdb_txn, NULL, MDB_DUPSORT, &mdb_dbi);
if (retVal != 0) {
mdb_txn_commit(mdb_txn);
return 0;
}
// prepare data
db_key.mv_size = key_size;
db_key.mv_data = (char*)key;
retVal = mdb_get(mdb_txn, mdb_dbi, &db_key, &db_value);
if (retVal != 0) {
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 0;
}
// now copy the data
if (db_value.mv_size > max_data_size) {
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 0;
}
// set return values
memcpy(data, db_value.mv_data, db_value.mv_size);
(*data_size) = db_value.mv_size;
// clean up
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 1;
}
/**
* Write data to the datastore with the specified key
* @param key the key
* @param key_size the length of the key
* @param data the data to be written
* @param data_size the length of the data to be written
* @param datastore the datastore to write to
* @returns true(1) on success
*/
int repo_fsrepo_lmdb_put(unsigned const char* key, size_t key_size, unsigned char* data, size_t data_size, const struct Datastore* datastore) {
int retVal; int retVal;
MDB_txn* mdb_txn; MDB_txn* mdb_txn;
MDB_dbi mdb_dbi; MDB_dbi mdb_dbi;
@ -58,7 +168,16 @@ int repo_fsrepo_lmdb_put(const char* key, size_t key_size, unsigned char* data,
return retVal; return retVal;
} }
/**
* Write a block to the datastore with the specified key
* @param block the block to be written
* @param datastore the datastore to write to
* @returns true(1) on success
*/
int repo_fsrepo_lmdb_put_block(const struct Block* block, const struct Datastore* datastore) {
return repo_fsrepo_lmdb_put(block->cid->hash, block->cid->hash_length, block->data, block->data_length, datastore);
}
/** /**
* Open an lmdb database with the given parameters. * Open an lmdb database with the given parameters.
@ -108,7 +227,9 @@ int repo_fsrepo_lmdb_cast(struct Datastore* datastore) {
datastore->datastore_open = &repo_fsrepro_lmdb_open; datastore->datastore_open = &repo_fsrepro_lmdb_open;
datastore->datastore_close = &repo_fsrepo_lmdb_close; datastore->datastore_close = &repo_fsrepo_lmdb_close;
datastore->datastore_put = &repo_fsrepo_lmdb_put; datastore->datastore_put = &repo_fsrepo_lmdb_put;
//datastore->datastore_get = &repo_fsrepo_lmdb_get; datastore->datastore_put_block = &repo_fsrepo_lmdb_put_block;
datastore->datastore_get = &repo_fsrepo_lmdb_get;
datastore->datastore_get_block = &repo_fsrepo_lmdb_get_block;
return 1; return 1;
} }

View file

@ -2,6 +2,68 @@
#include "ipfs/node/node.h" #include "ipfs/node/node.h"
#include "../test_helper.h" #include "../test_helper.h"
int test_merkledag_get_data() {
int retVal = 0;
// create a fresh repo
retVal = drop_and_build_repository("/tmp/.ipfs");
if (retVal == 0)
return 0;
// open the fs repo
struct RepoConfig* repo_config = NULL;
struct FSRepo* fs_repo;
const char* path = "/tmp/.ipfs";
// create the struct
retVal = ipfs_repo_fsrepo_new((char*)path, repo_config, &fs_repo);
if (retVal == 0)
return 0;
// open the repository and read the config file
retVal = ipfs_repo_fsrepo_open(fs_repo);
if (retVal == 0) {
ipfs_repo_fsrepo_free(fs_repo);
return 0;
}
// get the size of the database
int start_file_size = os_utils_file_size("/tmp/.ipfs/datastore/data.mdb");
// create data for node
size_t binary_data_size = 256;
unsigned char binary_data[binary_data_size];
for(int i = 0; i < binary_data_size; i++) {
binary_data[i] = i;
}
// create a node
struct Node* node1 = N_Create_From_Data(binary_data, 256);
retVal = ipfs_merkledag_add(node1, fs_repo);
if (retVal == 0) {
Node_Delete(node1);
return 0;
}
// now retrieve it
struct Node* results_node;
retVal = ipfs_merkledag_get(node1->cached, &results_node, fs_repo);
if (retVal == 0)
return 0;
if (results_node->data_size != 256)
return 0;
// the data should be the same
for(int i = 0; i < results_node->data_size; i++) {
if (results_node->data[i] != node1->data[i])
return 0;
}
return retVal;
}
int test_merkledag_add_data() { int test_merkledag_add_data() {
int retVal = 0; int retVal = 0;
@ -31,14 +93,14 @@ int test_merkledag_add_data() {
int start_file_size = os_utils_file_size("/tmp/.ipfs/datastore/data.mdb"); int start_file_size = os_utils_file_size("/tmp/.ipfs/datastore/data.mdb");
// create data for node // create data for node
size_t binary_data_size = 255; size_t binary_data_size = 256;
unsigned char binary_data[binary_data_size]; unsigned char binary_data[binary_data_size];
for(int i = 0; i < binary_data_size; i++) { for(int i = 0; i < binary_data_size; i++) {
binary_data[i] = i; binary_data[i] = i;
} }
// create a node // create a node
struct Node* node1 = N_Create_From_Data(binary_data, 255); struct Node* node1 = N_Create_From_Data(binary_data, 256);
retVal = ipfs_merkledag_add(node1, fs_repo); retVal = ipfs_merkledag_add(node1, fs_repo);
if (retVal == 0) { if (retVal == 0) {

View file

@ -1,18 +1,18 @@
#include "ipfs/blocks/block.h" #include "ipfs/blocks/block.h"
int test_blocks_new() { int test_blocks_new() {
const char* input = "Hello, World!"; const unsigned char* input = (const unsigned char*)"Hello, World!";
int retVal = 0; int retVal = 0;
struct Block* block; struct Block* block;
retVal = ipfs_blocks_block_new(input, strlen(input) + 1, &block); retVal = ipfs_blocks_block_new(input, strlen((const char*)input) + 1, &block);
if (retVal == 0) if (retVal == 0)
return 0; return 0;
// now examine the block // now examine the block
if (strcmp(block->data, input) != 0) if (strcmp((const char*)block->data, (const char*)input) != 0)
return 0; return 0;
if (block->data_length != strlen(input) + 1) if (block->data_length != strlen((const char*)input) + 1)
return 0; return 0;
if (block->cid->codec != CID_PROTOBUF) if (block->cid->codec != CID_PROTOBUF)

View file

@ -14,8 +14,8 @@
*/ */
int test_ipfs_datastore_put() { int test_ipfs_datastore_put() {
struct Block* block; struct Block* block;
int retVal; int retVal = 0;
const unsigned char* input = "Hello, world!"; const unsigned char* input = (unsigned char*)"Hello, world!";
// build the ipfs repository, then shut it down, so we can start fresh // build the ipfs repository, then shut it down, so we can start fresh
drop_and_build_repository("/tmp/.ipfs"); drop_and_build_repository("/tmp/.ipfs");
@ -44,7 +44,7 @@ int test_ipfs_datastore_put() {
return 0; return 0;
// send to Put with key // send to Put with key
retVal = fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore); retVal = fs_repo->config->datastore->datastore_put((const unsigned char*)key, key_length, block->data, block->data_length, fs_repo->config->datastore);
if (retVal == 0) if (retVal == 0)
return 0; return 0;

View file

@ -1,7 +1,9 @@
#include <dirent.h> #include <dirent.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <string.h>
#include "ipfs/repo/fsrepo/fs_repo.h" #include "ipfs/repo/fsrepo/fs_repo.h"
#include "ipfs/os/utils.h"
int remove_directory(const char *path) int remove_directory(const char *path)
{ {

View file

@ -41,7 +41,8 @@ const char* names[] = {
"test_repo_bootstrap_peers_init", "test_repo_bootstrap_peers_init",
"test_ipfs_datastore_put", "test_ipfs_datastore_put",
"test_node", "test_node",
"test_merkledag_add_data" "test_merkledag_add_data",
"test_merkledag_get_data"
}; };
int (*funcs[])(void) = { int (*funcs[])(void) = {
@ -64,7 +65,8 @@ int (*funcs[])(void) = {
test_repo_bootstrap_peers_init, test_repo_bootstrap_peers_init,
test_ipfs_datastore_put, test_ipfs_datastore_put,
test_node, test_node,
test_merkledag_add_data test_merkledag_add_data,
test_merkledag_get_data
}; };
/** /**