agorise/c-ipfs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Implementation of MerkleDag get and put

Browse source 
Now saving and retrieving MerkleDags that contain data. Now need to work
with links and other types.
This commit is contained in:
jmjatlanta 2016-12-05 17:23:58 -05:00
parent 8a80d2afc7
commit da6490ac7f
14 changed files with 246 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
include/ipfs/merkledag/merkledag.h
View file

@ -12,5 +12,13 @@
*/
int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo);
/***
* Retrieves a node from the datastore based on the cid
* @param cid the key to look for
* @param node the node to be created
* @param fs_repo the repository
* @returns true(1) on success
*/
int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo);
#endif

2
include/ipfs/repo/config/config.h
View file

@ -72,7 +72,7 @@ int config_path(char* config_root, char* extension, char* result, int max_len);
* @param num_bits_for_keypair number of bits for the key pair
* @returns true(1) on success, otherwise 0
*/
int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path);
int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, const char* repo_path);
/***
* Initialize memory for a RepoConfig struct

8
include/ipfs/repo/config/datastore.h
View file

@ -20,8 +20,10 @@ struct Datastore {
// function pointers for datastore operations
int (*datastore_open)(int argc, char** argv, struct Datastore* datastore);
int (*datastore_close)(struct Datastore* datastore);
int (*datastore_put)(const char* key, size_t key_size, unsigned char* data, size_t data_length, struct Datastore* datastore);
//int (*datastore_get)(const char* key, struct Block* block);
int (*datastore_put)(const unsigned char* key, size_t key_size, unsigned char* data, size_t data_length, const struct Datastore* datastore);
int (*datastore_put_block)(const struct Block* block, const struct Datastore* datastore);
int (*datastore_get)(const char* key, size_t key_size, unsigned char* data, size_t max_data_length, size_t* data_length, const struct Datastore* datastore);
int (*datastore_get_block)(const struct Cid* cid, struct Block** block, const struct Datastore* datastore);
// a handle to the datastore "context" used by the datastore
void* handle;
};
@ -33,7 +35,7 @@ struct Datastore {
* @param config_root the path to the root of IPFS
* @returns true(1) on success
*/
int ipfs_repo_config_datastore_init(struct Datastore* datastore, char* config_root);
int ipfs_repo_config_datastore_init(struct Datastore* datastore, const char* config_root);
/***
* initialize the structure of the datastore

2
include/ipfs/repo/fsrepo/fs_repo.h
View file

@ -48,7 +48,7 @@ int fs_repo_write_config_file(char* path, struct RepoConfig* config);
* @param repo the struct to allocate memory for
* @returns false(0) if something bad happened, otherwise true(1)
*/
int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** fs_repo);
int ipfs_repo_fsrepo_new(const char* repo_path, struct RepoConfig* config, struct FSRepo** fs_repo);
/***
* Free all resources used by this struct

25
merkledag/merkledag.c
View file

@ -17,7 +17,7 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
struct Block* block;
ipfs_blocks_block_new(node->data, node->data_size, &block);
int retVal = fs_repo->config->datastore->datastore_put(block->cid->hash, block->cid->hash_length, block->data, block->data_length, fs_repo->config->datastore);
int retVal = fs_repo->config->datastore->datastore_put_block(block, fs_repo->config->datastore);
if (retVal == 0) {
ipfs_blocks_block_free(block);
return 0;
@ -29,3 +29,26 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
// TODO: call HasBlock (unsure why as yet)
return 1;
}
/***
* Retrieves a node from the datastore based on the cid
* @param cid the key to look for
* @param node the node to be created
* @param fs_repo the repository
* @returns true(1) on success
*/
int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo) {
int retVal = 1;
// look for the block
struct Block* block;
retVal = fs_repo->config->datastore->datastore_get_block(cid, &block, fs_repo->config->datastore);
if (retVal == 0)
return 0;
// we have the block. Fill the node
*node = N_Create_From_Data(block->data, block->data_length);
Node_Set_Cid(*node, cid);
return retVal;
}

2
repo/config/config.c
View file

@ -79,7 +79,7 @@ int repo_config_get_file_name(char* path, char** result) {
* @param num_bits_for_keypair number of bits for the key pair
* @returns true(1) on success, otherwise 0
*/
int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, char* repo_path) {
int ipfs_repo_config_init(struct RepoConfig* config, unsigned int num_bits_for_keypair, const char* repo_path) {
// identity
int retVal = repo_config_identity_init(config->identity, num_bits_for_keypair);
if (retVal == 0)

2
repo/config/datastore.c
View file

@ -17,7 +17,7 @@ int alloc_and_assign(char** result, const char* string) {
* @param datastore the struct to initialize
* @returns true(1) on success
*/
int ipfs_repo_config_datastore_init(struct Datastore* datastore, char* config_root) {
int ipfs_repo_config_datastore_init(struct Datastore* datastore, const char* config_root) {
unsigned long stringLength = strlen(config_root) + 12;
datastore->path = malloc(sizeof(char) * stringLength);
os_utils_filepath_join(config_root, "datastore", datastore->path, stringLength);

2
repo/fsrepo/fs_repo.c
View file

@ -108,7 +108,7 @@ int repo_config_write_config_file(char* full_filename, struct RepoConfig* config
* @param repo the struct to allocate memory for
* @returns false(0) if something bad happened, otherwise true(1)
*/
int ipfs_repo_fsrepo_new(char* repo_path, struct RepoConfig* config, struct FSRepo** repo) {
int ipfs_repo_fsrepo_new(const char* repo_path, struct RepoConfig* config, struct FSRepo** repo) {
*repo = (struct FSRepo*)malloc(sizeof(struct FSRepo));
if (repo_path == NULL) {

133
repo/fsrepo/lmdb_datastore.c
View file

@ -10,13 +10,123 @@
#include "lmdb.h"
#include "ipfs/repo/fsrepo/lmdb_datastore.h"
/**
* Write a block to the datastore with the specified key
* @param key the key
* @param block the block to be written
int repo_fsrepo_lmdb_get_block(const struct Cid* cid, struct Block** block, const struct Datastore* datastore) {
int retVal;
MDB_txn* mdb_txn;
MDB_dbi mdb_dbi;
struct MDB_val db_key;
struct MDB_val db_value;
MDB_env* mdb_env = (MDB_env*)datastore->handle;
if (mdb_env == NULL)
return 0;
// open transaction
retVal = mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn);
if (retVal != 0)
return 0;
retVal = mdb_dbi_open(mdb_txn, NULL, MDB_DUPSORT, &mdb_dbi);
if (retVal != 0) {
mdb_txn_commit(mdb_txn);
return 0;
}
// prepare data
db_key.mv_size = cid->hash_length;
db_key.mv_data = cid->hash;
retVal = mdb_get(mdb_txn, mdb_dbi, &db_key, &db_value);
if (retVal != 0) {
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 0;
}
// now copy the data
retVal = ipfs_blocks_block_new(db_value.mv_data, db_value.mv_size, block);
if (retVal == 0) {
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 0;
}
// clean up
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 1;
}
/***
* retrieve a record from the database and put in a pre-sized buffer
* @param key the key to look for
* @param key_size the length of the key
* @param data the data that is retrieved
* @param max_data_size the length of the data buffer
* @param data_size the length of the data that was found in the database
* @param datastore where to look for the data
* @returns true(1) on success
*/
int repo_fsrepo_lmdb_put(const char* key, size_t key_size, unsigned char* data, size_t data_size, struct Datastore* datastore) {
int repo_fsrepo_lmdb_get(const char* key, size_t key_size, unsigned char* data, size_t max_data_size, size_t* data_size, const struct Datastore* datastore) {
int retVal;
MDB_txn* mdb_txn;
MDB_dbi mdb_dbi;
struct MDB_val db_key;
struct MDB_val db_value;
MDB_env* mdb_env = (MDB_env*)datastore->handle;
if (mdb_env == NULL)
return 0;
// open transaction
retVal = mdb_txn_begin(mdb_env, NULL, 0, &mdb_txn);
if (retVal != 0)
return 0;
retVal = mdb_dbi_open(mdb_txn, NULL, MDB_DUPSORT, &mdb_dbi);
if (retVal != 0) {
mdb_txn_commit(mdb_txn);
return 0;
}
// prepare data
db_key.mv_size = key_size;
db_key.mv_data = (char*)key;
retVal = mdb_get(mdb_txn, mdb_dbi, &db_key, &db_value);
if (retVal != 0) {
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 0;
}
// now copy the data
if (db_value.mv_size > max_data_size) {
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 0;
}
// set return values
memcpy(data, db_value.mv_data, db_value.mv_size);
(*data_size) = db_value.mv_size;
// clean up
mdb_dbi_close(mdb_env, mdb_dbi);
mdb_txn_commit(mdb_txn);
return 1;
}
/**
* Write data to the datastore with the specified key
* @param key the key
* @param key_size the length of the key
* @param data the data to be written
* @param data_size the length of the data to be written
* @param datastore the datastore to write to
* @returns true(1) on success
*/
int repo_fsrepo_lmdb_put(unsigned const char* key, size_t key_size, unsigned char* data, size_t data_size, const struct Datastore* datastore) {
int retVal;
MDB_txn* mdb_txn;
MDB_dbi mdb_dbi;
@ -58,7 +168,16 @@ int repo_fsrepo_lmdb_put(const char* key, size_t key_size, unsigned char* data,
return retVal;
}
/**
* Write a block to the datastore with the specified key
* @param block the block to be written
* @param datastore the datastore to write to
* @returns true(1) on success
*/
int repo_fsrepo_lmdb_put_block(const struct Block* block, const struct Datastore* datastore) {
return repo_fsrepo_lmdb_put(block->cid->hash, block->cid->hash_length, block->data, block->data_length, datastore);
}
/**
* Open an lmdb database with the given parameters.
@ -108,7 +227,9 @@ int repo_fsrepo_lmdb_cast(struct Datastore* datastore) {
datastore->datastore_open = &repo_fsrepro_lmdb_open;
datastore->datastore_close = &repo_fsrepo_lmdb_close;
datastore->datastore_put = &repo_fsrepo_lmdb_put;
//datastore->datastore_get = &repo_fsrepo_lmdb_get;
datastore->datastore_put_block = &repo_fsrepo_lmdb_put_block;
datastore->datastore_get = &repo_fsrepo_lmdb_get;
datastore->datastore_get_block = &repo_fsrepo_lmdb_get_block;
return 1;
}

66
test/merkledag/test_merkledag.h
View file

@ -2,6 +2,68 @@
#include "ipfs/node/node.h"
#include "../test_helper.h"
int test_merkledag_get_data() {
int retVal = 0;
// create a fresh repo
retVal = drop_and_build_repository("/tmp/.ipfs");
if (retVal == 0)
return 0;
// open the fs repo
struct RepoConfig* repo_config = NULL;
struct FSRepo* fs_repo;
const char* path = "/tmp/.ipfs";
// create the struct
retVal = ipfs_repo_fsrepo_new((char*)path, repo_config, &fs_repo);
if (retVal == 0)
return 0;
// open the repository and read the config file
retVal = ipfs_repo_fsrepo_open(fs_repo);
if (retVal == 0) {
ipfs_repo_fsrepo_free(fs_repo);
return 0;
}
// get the size of the database
int start_file_size = os_utils_file_size("/tmp/.ipfs/datastore/data.mdb");
// create data for node
size_t binary_data_size = 256;
unsigned char binary_data[binary_data_size];
for(int i = 0; i < binary_data_size; i++) {
binary_data[i] = i;
}
// create a node
struct Node* node1 = N_Create_From_Data(binary_data, 256);
retVal = ipfs_merkledag_add(node1, fs_repo);
if (retVal == 0) {
Node_Delete(node1);
return 0;
}
// now retrieve it
struct Node* results_node;
retVal = ipfs_merkledag_get(node1->cached, &results_node, fs_repo);
if (retVal == 0)
return 0;
if (results_node->data_size != 256)
return 0;
// the data should be the same
for(int i = 0; i < results_node->data_size; i++) {
if (results_node->data[i] != node1->data[i])
return 0;
}
return retVal;
}
int test_merkledag_add_data() {
int retVal = 0;
@ -31,14 +93,14 @@ int test_merkledag_add_data() {
int start_file_size = os_utils_file_size("/tmp/.ipfs/datastore/data.mdb");
// create data for node
size_t binary_data_size = 255;
size_t binary_data_size = 256;
unsigned char binary_data[binary_data_size];
for(int i = 0; i < binary_data_size; i++) {
binary_data[i] = i;
}
// create a node
struct Node* node1 = N_Create_From_Data(binary_data, 255);
struct Node* node1 = N_Create_From_Data(binary_data, 256);
retVal = ipfs_merkledag_add(node1, fs_repo);
if (retVal == 0) {

8
test/storage/test_blocks.h
View file

@ -1,18 +1,18 @@
#include "ipfs/blocks/block.h"
int test_blocks_new() {
const char* input = "Hello, World!";
const unsigned char* input = (const unsigned char*)"Hello, World!";
int retVal = 0;
struct Block* block;
retVal = ipfs_blocks_block_new(input, strlen(input) + 1, &block);
retVal = ipfs_blocks_block_new(input, strlen((const char*)input) + 1, &block);
if (retVal == 0)
return 0;
// now examine the block
if (strcmp(block->data, input) != 0)
if (strcmp((const char*)block->data, (const char*)input) != 0)
return 0;
if (block->data_length != strlen(input) + 1)
if (block->data_length != strlen((const char*)input) + 1)
return 0;
if (block->cid->codec != CID_PROTOBUF)

6
test/storage/test_datastore.h
View file

@ -14,8 +14,8 @@
*/
int test_ipfs_datastore_put() {
struct Block* block;
int retVal;
const unsigned char* input = "Hello, world!";
int retVal = 0;
const unsigned char* input = (unsigned char*)"Hello, world!";
// build the ipfs repository, then shut it down, so we can start fresh
drop_and_build_repository("/tmp/.ipfs");
@ -44,7 +44,7 @@ int test_ipfs_datastore_put() {
return 0;
// send to Put with key
retVal = fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
retVal = fs_repo->config->datastore->datastore_put((const unsigned char*)key, key_length, block->data, block->data_length, fs_repo->config->datastore);
if (retVal == 0)
return 0;

2
test/test_helper.c
View file

@ -1,7 +1,9 @@
#include <dirent.h>
#include <sys/stat.h>
#include <string.h>
#include "ipfs/repo/fsrepo/fs_repo.h"
#include "ipfs/os/utils.h"
int remove_directory(const char *path)
{

6
test/testit.c
View file

@ -41,7 +41,8 @@ const char* names[] = {
"test_repo_bootstrap_peers_init",
"test_ipfs_datastore_put",
"test_node",
"test_merkledag_add_data"
"test_merkledag_add_data",
"test_merkledag_get_data"
};
int (*funcs[])(void) = {
@ -64,7 +65,8 @@ int (*funcs[])(void) = {
test_repo_bootstrap_peers_init,
test_ipfs_datastore_put,
test_node,
test_merkledag_add_data
test_merkledag_add_data,
test_merkledag_get_data
};
/**