diff --git a/blocks/block.c b/blocks/block.c index 4d6dcf6..c917526 100644 --- a/blocks/block.c +++ b/blocks/block.c @@ -9,6 +9,100 @@ #include "ipfs/blocks/block.h" #include "ipfs/cid/cid.h" +/*** + * The protobuf functions + */ +// protobuf fields: data & data_length cid +enum WireType ipfs_block_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED}; + +/** + * Determine the approximate size of an encoded block + * @param block the block to measure + * @returns the approximate size needed to encode the protobuf + */ +size_t ipfs_blocks_block_protobuf_encode_size(const struct Block* block) { + return 22 + ipfs_cid_protobuf_encode_size(block->cid) + block->data_length; +} + +/** + * Encode the Block into protobuf format + * @param block the block to encode + * @param buffer the buffer to fill + * @param max_buffer_size the max size of the buffer + * @param bytes_written the number of bytes used + * @returns true(1) on success + */ +int ipfs_blocks_block_protobuf_encode(const struct Block* block, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) { + // data & data_size + size_t bytes_used = 0; + *bytes_written = 0; + int retVal = 0; + retVal = protobuf_encode_length_delimited(1, ipfs_block_message_fields[0], (char*)block->data, block->data_length, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + *bytes_written += bytes_used; + // cid + size_t cid_size = ipfs_cid_protobuf_encode_size(block->cid); + unsigned char cid[cid_size]; + retVal = ipfs_cid_protobuf_encode(block->cid, cid, cid_size, &cid_size); + retVal = protobuf_encode_length_delimited(2, ipfs_block_message_fields[1], (char*)cid, cid_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + *bytes_written += bytes_used; + return 1; +} + +/*** + * Decode from a protobuf stream into a Block struct + * @param buffer the buffer to pull from + * @param buffer_length the length of the buffer + * @param block the block to fill + * @returns true(1) on success + */ +int ipfs_blocks_block_protobuf_decode(const unsigned char* buffer, const size_t buffer_length, struct Block** block) { + size_t pos = 0; + int retVal = 0; + unsigned char* temp_buffer = NULL; + size_t temp_size; + + if (ipfs_blocks_block_new(block) == 0) + goto exit; + + while(pos < buffer_length) { + size_t bytes_read = 0; + int field_no; + enum WireType field_type; + if (protobuf_decode_field_and_type(&buffer[pos], buffer_length, &field_no, &field_type, &bytes_read) == 0) { + goto exit; + } + pos += bytes_read; + switch(field_no) { + case (1): // data + if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*block)->data), &((*block)->data_length), &bytes_read) == 0) + goto exit; + pos += bytes_read; + break; + case (2): // cid + if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0) + goto exit; + pos += bytes_read; + if (ipfs_cid_protobuf_decode(temp_buffer, temp_size, &((*block)->cid)) == 0) + goto exit; + free(temp_buffer); + temp_buffer = NULL; + break; + } + } + + retVal = 1; + +exit: + if (retVal == 0) { + ipfs_blocks_block_free(*block); + } + if (temp_buffer != NULL) + free(temp_buffer); + + return retVal; +} + + /*** * Create a new block based on the incoming data * @param data the data to base the block on @@ -16,35 +110,38 @@ * @param block a pointer to the struct Block that will be created * @returns true(1) on success */ -int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block) { +int ipfs_blocks_block_new(struct Block** block) { // allocate memory for structure (*block) = (struct Block*)malloc(sizeof(struct Block)); if ((*block) == NULL) return 0; + (*block)->data = NULL; + (*block)->data_length = 0; + return 1; +} + +int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, struct Block* block) { // cid unsigned char hash[32]; if (libp2p_crypto_hashing_sha256(data, data_size, &hash[0]) == 0) { - free(*block); return 0; } - if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &((*block)->cid)) == 0) { - free(*block); + if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &(block->cid)) == 0) { return 0; } - (*block)->data_length = data_size; + block->data_length = data_size; - (*block)->data = malloc(sizeof(unsigned char) * data_size); - if ( (*block)->data == NULL) { - ipfs_cid_free((*block)->cid); - free(*block); + block->data = malloc(sizeof(unsigned char) * data_size); + if ( block->data == NULL) { + ipfs_cid_free(block->cid); return 0; } - memcpy( (*block)->data, data, data_size); + memcpy( block->data, data, data_size); return 1; } diff --git a/blocks/blockstore.c b/blocks/blockstore.c index d9ffda8..99823ae 100644 --- a/blocks/blockstore.c +++ b/blocks/blockstore.c @@ -6,6 +6,7 @@ #include "ipfs/blocks/block.h" #include "ipfs/datastore/ds_helper.h" #include "ipfs/repo/fsrepo/fs_repo.h" +#include "ipfs/os/utils.h" /** * Delete a block based on its Cid @@ -25,14 +26,59 @@ int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo) { return 0; } +unsigned char* ipfs_blockstore_cid_to_base32(const struct Cid* cid) { + size_t key_length = libp2p_crypto_encoding_base32_encode_size(cid->hash_length); + unsigned char* buffer = (unsigned char*)malloc(key_length + 1); + if (buffer == NULL) + return NULL; + int retVal = ipfs_datastore_helper_ds_key_from_binary(cid->hash, cid->hash_length, &buffer[0], key_length, &key_length); + if (retVal == 0) { + free(buffer); + return NULL; + } + buffer[key_length] = 0; + return buffer; +} + +char* ipfs_blockstore_path_get(const struct FSRepo* fs_repo, const char* filename) { + int filepath_size = strlen(fs_repo->path) + 12; + char filepath[filepath_size]; + int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", filepath, filepath_size); + if (retVal == 0) { + free(filepath); + return 0; + } + int complete_filename_size = strlen(filepath) + strlen(filename) + 2; + char* complete_filename = (char*)malloc(complete_filename_size); + retVal = os_utils_filepath_join(filepath, filename, complete_filename, complete_filename_size); + return complete_filename; +} + /*** * Find a block based on its Cid * @param cid the Cid to look for * @param block where to put the data to be returned * @returns true(1) on success */ -int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo) { - return 0; +int ipfs_blockstore_get(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo) { + // get datastore key, which is a base32 key of the multihash + unsigned char* key = ipfs_blockstore_cid_to_base32(cid); + + char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key); + + size_t file_size = os_utils_file_size(filename); + unsigned char buffer[file_size]; + + FILE* file = fopen(filename, "rb"); + size_t bytes_read = fread(buffer, 1, file_size, file); + fclose(file); + + int retVal = ipfs_blocks_block_protobuf_decode(buffer, bytes_read, block); + + free(key); + free(filename); + + return retVal; } /*** @@ -42,14 +88,46 @@ int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_ */ int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) { // from blockstore.go line 118 - // Get Datastore key, which is a base32 key of the binary, - size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length); - unsigned char key[key_length]; - int retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length); - if (retVal == 0) - return 0; + int retVal = 0; - // send to Put with key - fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore); + // Get Datastore key, which is a base32 key of the multihash, + unsigned char* key = ipfs_blockstore_cid_to_base32(block->cid); + if (key == NULL) { + free(key); + return 0; + } + + //TODO: put this in subdirectories + + // turn the block into a binary array + size_t protobuf_len = ipfs_blocks_block_protobuf_encode_size(block); + unsigned char protobuf[protobuf_len]; + retVal = ipfs_blocks_block_protobuf_encode(block, protobuf, protobuf_len, &protobuf_len); + if (retVal == 0) { + free(key); + return 0; + } + + // now write byte array to file + char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key); + if (filename == NULL) { + free(key); + return 0; + } + + FILE* file = fopen(filename, "wb"); + int bytes_written = fwrite(protobuf, 1, protobuf_len, file); + fclose(file); + if (bytes_written != protobuf_len) { + free(key); + free(filename); + return 0; + } + + // send to Put with key (this is now done separately) + //fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore); + + free(key); + free(filename); return 1; } diff --git a/datastore/ds_helper.c b/datastore/ds_helper.c index 2e1a75d..c78f483 100644 --- a/datastore/ds_helper.c +++ b/datastore/ds_helper.c @@ -1,10 +1,11 @@ /** * Some code to help with the datastore / blockstore interface + * NOTE: the datastore stores things under a multihash key */ #include "libp2p/crypto/encoding/base32.h" #include "ipfs/datastore/ds_helper.h" /** - * Generate a key based on the passed in binary_array + * Generate a base32 key based on the passed in binary_array (which is normally a multihash) * @param binary_array what to base the key on * @param array_length the size of the binary array * @param results where the key will be put @@ -30,7 +31,7 @@ int ipfs_datastore_helper_ds_key_from_binary(unsigned char* binary_array, size_t } /** - * Generate a binary array based on the passed in datastore key + * Generate a binary array (normally a multihash) based on the passed in datastore key * @param ds_key the base32 encoded key * @param key_length the length of the base32 "string" * @param binary_array where to put the decoded value diff --git a/importer/importer.c b/importer/importer.c index 585185f..524a803 100644 --- a/importer/importer.c +++ b/importer/importer.c @@ -1,6 +1,7 @@ #include #include "ipfs/importer/importer.h" +#include "ipfs/merkledag/merkledag.h" #define MAX_DATA_SIZE 262144 // 1024 * 256; @@ -14,26 +15,27 @@ * @param node the node to add to * @returns number of bytes read */ -size_t ipfs_import_chunk(FILE* file, struct Node* node) { +size_t ipfs_import_chunk(FILE* file, struct Node* node, struct FSRepo* fs_repo) { unsigned char buffer[MAX_DATA_SIZE]; - size_t bytes_read = fread(buffer, MAX_DATA_SIZE, 1, file); + size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file); if (node->data_size == 0) { ipfs_node_set_data(node, buffer, bytes_read); - if (bytes_read != MAX_DATA_SIZE) { - // persist - } } else { // create a new node, and link to the parent struct Node* new_node = NULL; ipfs_node_new_from_data(buffer, bytes_read, &new_node); // persist - + ipfs_merkledag_add(new_node, fs_repo); // put link in node struct NodeLink* new_link = NULL; ipfs_node_link_new("", new_node->cached->hash, &new_link); ipfs_node_add_link(node, new_link); ipfs_node_free(new_node); } + if (bytes_read != MAX_DATA_SIZE) { + // persist the main node + ipfs_merkledag_add(node, fs_repo); + } return bytes_read; } @@ -43,7 +45,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* node) { * @param node the root node (could have links to others) * @returns true(1) on success */ -int ipfs_import_file(const char* fileName, struct Node** node) { +int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo) { int retVal = 1; int bytes_read = MAX_DATA_SIZE; @@ -54,7 +56,7 @@ int ipfs_import_file(const char* fileName, struct Node** node) { // add all nodes while ( bytes_read == MAX_DATA_SIZE) { - bytes_read = ipfs_import_chunk(file, *node); + bytes_read = ipfs_import_chunk(file, *node, fs_repo); } fclose(file); diff --git a/include/ipfs/blocks/block.h b/include/ipfs/blocks/block.h index 288c3f0..9d27cdb 100644 --- a/include/ipfs/blocks/block.h +++ b/include/ipfs/blocks/block.h @@ -1,5 +1,6 @@ /*** * IPFS has the notion of storage blocks. + * Raw data with a multihash key (the Cid) */ #ifndef __IPFS_BLOCKS_BLOCK_H__ @@ -20,7 +21,9 @@ struct Block { * @param block a pointer to the struct Block that will be created * @returns true(1) on success */ -int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block); +int ipfs_blocks_block_new(struct Block** block); + +int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, struct Block* block); /*** * Free resources used by the creation of a block @@ -29,4 +32,30 @@ int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Bl */ int ipfs_blocks_block_free(struct Block* block); +/** + * Determine the approximate size of an encoded block + * @param block the block to measure + * @returns the approximate size needed to encode the protobuf + */ +size_t ipfs_blocks_block_protobuf_encode_size(const struct Block* block); + +/** + * Encode the Block into protobuf format + * @param block the block to encode + * @param buffer the buffer to fill + * @param max_buffer_size the max size of the buffer + * @param bytes_written the number of bytes used + * @returns true(1) on success + */ +int ipfs_blocks_block_protobuf_encode(const struct Block* block, unsigned char* buffer, size_t max_buffer_size, size_t* bytes_written); + +/*** + * Decode from a protobuf stream into a Block struct + * @param buffer the buffer to pull from + * @param buffer_length the length of the buffer + * @param block the block to fill + * @returns true(1) on success + */ +int ipfs_blocks_block_protobuf_decode(const unsigned char* buffer, const size_t buffer_length, struct Block** block); + #endif diff --git a/include/ipfs/blocks/blockstore.h b/include/ipfs/blocks/blockstore.h index f102c28..37f2569 100644 --- a/include/ipfs/blocks/blockstore.h +++ b/include/ipfs/blocks/blockstore.h @@ -3,7 +3,10 @@ */ #ifndef __IPFS_BLOCKS_BLOCKSTORE_H__ -#ifndef __IPFS_BLOCKS_BLOCKSTORE_H__ +#define __IPFS_BLOCKS_BLOCKSTORE_H__ + +#include "ipfs/cid/cid.h" +#include "ipfs/repo/fsrepo/fs_repo.h" /** * Delete a block based on its Cid @@ -25,14 +28,14 @@ int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo); * @param block where to put the data to be returned * @returns true(1) on success */ -int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo); +int ipfs_blockstore_get(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo); /*** * Put a block in the blockstore * @param block the block to store * @returns true(1) on success */ -int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo); +int ipfs_blockstore_put(struct Block* block, const struct FSRepo* fs_repo); #endif diff --git a/include/ipfs/importer/importer.h b/include/ipfs/importer/importer.h index 01e44d3..b1292a7 100644 --- a/include/ipfs/importer/importer.h +++ b/include/ipfs/importer/importer.h @@ -2,6 +2,7 @@ #define __IPFS_IMPORTER_IMPORTER_H__ #include "ipfs/node/node.h" +#include "ipfs/repo/fsrepo/fs_repo.h" /** * Creates a node based on an incoming file @@ -9,6 +10,6 @@ * @param node the root node (could have links to others) * @returns true(1) on success */ -int ipfs_import_file(const char* fileName, struct Node** node); +int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo); #endif /* INCLUDE_IPFS_IMPORTER_IMPORTER_H_ */ diff --git a/include/ipfs/repo/fsrepo/fs_repo.h b/include/ipfs/repo/fsrepo/fs_repo.h index c866ce2..7ab6d2c 100644 --- a/include/ipfs/repo/fsrepo/fs_repo.h +++ b/include/ipfs/repo/fsrepo/fs_repo.h @@ -64,4 +64,13 @@ int ipfs_repo_fsrepo_free(struct FSRepo* config); */ int ipfs_repo_fsrepo_init(struct FSRepo* config); +/*** + * Write a block to the datastore and blockstore + * @param block the block to write + * @param fs_repo the repo to write to + * @returns true(1) on success + */ +int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo); +int ipfs_repo_fsrepo_block_read(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo); + #endif /* fs_repo_h */ diff --git a/merkledag/merkledag.c b/merkledag/merkledag.c index e9a0bc4..0753fd2 100644 --- a/merkledag/merkledag.c +++ b/merkledag/merkledag.c @@ -21,9 +21,11 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) { // turn the node into a block struct Block* block; - ipfs_blocks_block_new(protobuf, bytes_written, &block); + ipfs_blocks_block_new(&block); + ipfs_blocks_block_add_data(protobuf, bytes_written, block); - int retVal = fs_repo->config->datastore->datastore_put_block(block, fs_repo->config->datastore); + // write to block store & datastore + int retVal = ipfs_repo_fsrepo_block_write(block, fs_repo); if (retVal == 0) { ipfs_blocks_block_free(block); return 0; @@ -45,17 +47,25 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) { */ int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo) { int retVal = 1; - - // look for the block struct Block* block; - retVal = fs_repo->config->datastore->datastore_get_block(cid, &block, fs_repo->config->datastore); + size_t key_length = 100; + unsigned char key[key_length]; + + // look for the node in the datastore. If it is not there, it is not a node. + // If it exists, it is only a block. + retVal = fs_repo->config->datastore->datastore_get((char*)cid->hash, cid->hash_length, key, key_length, &key_length, fs_repo->config->datastore); if (retVal == 0) return 0; - // we have the block. Fill the node + // we have the record from the db. Go get the block from the blockstore + retVal = ipfs_repo_fsrepo_block_read(cid, &block, fs_repo); + + // now convert the block into a node ipfs_node_protobuf_decode(block->data, block->data_length, node); + // doesn't decode do this? ipfs_node_set_cached(*node, cid); + // free resources ipfs_blocks_block_free(block); return retVal; diff --git a/node/node.c b/node/node.c index 8de07b8..ac964d7 100644 --- a/node/node.c +++ b/node/node.c @@ -295,6 +295,7 @@ int ipfs_node_new(struct Node** node) return 0; (*node)->cached = NULL; (*node)->data = NULL; + (*node)->data_size = 0; (*node)->encoded = NULL; (*node)->head_link = NULL; return 1; diff --git a/os/utils.c b/os/utils.c index e35f097..41dc01c 100644 --- a/os/utils.c +++ b/os/utils.c @@ -62,11 +62,14 @@ int os_utils_directory_writeable(const char* path) { } int os_utils_file_size(const char* path) { + size_t file_size = 0; // open file FILE* in_file = fopen(path, "r"); + if (in_file == NULL) + return 0; // determine size fseek(in_file, 0L, SEEK_END); - size_t file_size = ftell(in_file); + file_size = ftell(in_file); fclose(in_file); return file_size; } diff --git a/repo/fsrepo/fs_repo.c b/repo/fsrepo/fs_repo.c index 5117ab6..0d8ae7c 100644 --- a/repo/fsrepo/fs_repo.c +++ b/repo/fsrepo/fs_repo.c @@ -1,6 +1,9 @@ #include +#include +#include "ipfs/blocks/blockstore.h" #include "libp2p/crypto/encoding/base64.h" +#include "ipfs/datastore/ds_helper.h" #include "ipfs/repo/config/datastore.h" #include "ipfs/repo/fsrepo/fs_repo.h" #include "ipfs/os/utils.h" @@ -460,6 +463,18 @@ int ipfs_repo_fsrepo_datastore_init(struct FSRepo* fs_repo) { return repo_fsrepo_lmdb_cast(fs_repo->config->datastore); } +int ipfs_repo_fsrepo_blockstore_init(const struct FSRepo* fs_repo) { + size_t full_path_size = strlen(fs_repo->path) + 15; + char full_path[full_path_size]; + int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", full_path, full_path_size); + if (retVal == 0) + return 0; + + if (mkdir(full_path, S_IRWXU) != 0) + return 0; + return 1; +} + /** * Initializes a new FSRepo at the given path with the provided config * @param path the path to use @@ -477,11 +492,14 @@ int ipfs_repo_fsrepo_init(struct FSRepo* repo) { if (retVal == 0) return 0; - // TODO: Implement this method retVal = ipfs_repo_fsrepo_datastore_init(repo); if (retVal == 0) return 0; + retVal = ipfs_repo_fsrepo_blockstore_init(repo); + if (retVal == 0) + return 0; + // write the version to a file for migrations (see repo/fsrepo/migrations/mfsr.go) //TODO: mfsr.RepoPath(repo_path).WriteVersion(RepoVersion) return 1; @@ -508,3 +526,46 @@ int fs_repo_write_config_file(char* path, struct RepoConfig* config) { return retVal; } +/*** + * Write a block to the datastore and blockstore + * @param block the block to write + * @param fs_repo the repo to write to + * @returns true(1) on success + */ +int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo) { + /** + * What is put in the blockstore is the block. + * What is put in the datastore is the multihash (the Cid) as the key, + * and the base32 encoded multihash as the value. + */ + int retVal = 1; + retVal = ipfs_blockstore_put(block, fs_repo); + if (retVal == 0) + return 0; + // take the cid, base32 it, and send both to the datastore + size_t fs_key_length = 100; + unsigned char fs_key[fs_key_length]; + retVal = ipfs_datastore_helper_ds_key_from_binary(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, &fs_key_length); + if (retVal == 0) + return 0; + retVal = fs_repo->config->datastore->datastore_put(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, fs_repo->config->datastore); + if (retVal == 0) + return 0; + return 1; +} + +int ipfs_repo_fsrepo_block_read(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo) { + int retVal = 0; + + // get the base32 hash from the database + // We do this only to see if it is in the database + size_t fs_key_length = 100; + unsigned char fs_key[fs_key_length]; + retVal = fs_repo->config->datastore->datastore_get((char*)cid->hash, cid->hash_length, fs_key, fs_key_length, &fs_key_length, fs_repo->config->datastore); + if (retVal == 0) // maybe it doesn't exist? + return 0; + // now get the block from the blockstore + retVal = ipfs_blockstore_get(cid, block, fs_repo); + return retVal; +} + diff --git a/repo/fsrepo/lmdb_datastore.c b/repo/fsrepo/lmdb_datastore.c index 1d632e0..9fc25de 100644 --- a/repo/fsrepo/lmdb_datastore.c +++ b/repo/fsrepo/lmdb_datastore.c @@ -1,5 +1,7 @@ /*** * Here are the wrappers for the lightning database + * NOTE: In this implementation, the database will contain the base32 encoded value + * of the multihash key if the file exists on disk. */ #include @@ -43,12 +45,13 @@ int repo_fsrepo_lmdb_get_block(const struct Cid* cid, struct Block** block, cons } // now copy the data - retVal = ipfs_blocks_block_new(db_value.mv_data, db_value.mv_size, block); + retVal = ipfs_blocks_block_new(block); if (retVal == 0) { mdb_dbi_close(mdb_env, mdb_dbi); mdb_txn_commit(mdb_txn); return 0; } + retVal = ipfs_blocks_block_add_data(db_value.mv_data, db_value.mv_size, *block); // clean up mdb_dbi_close(mdb_env, mdb_dbi); diff --git a/test/node/test_importer.h b/test/node/test_importer.h new file mode 100644 index 0000000..754a1c2 --- /dev/null +++ b/test/node/test_importer.h @@ -0,0 +1,77 @@ +#include + +#include "ipfs/importer/importer.h" +#include "ipfs/merkledag/merkledag.h" + +/*** + * Helper to create a test file in the OS + */ +int create_file(const char* fileName, unsigned char* bytes, size_t num_bytes) { + FILE* file = fopen(fileName, "wb"); + fwrite(bytes, num_bytes, 1, file); + fclose(file); + return 1; +} + +int create_bytes(unsigned char* buffer, size_t num_bytes) { + int counter = 0; + + for(int i = 0; i < num_bytes; i++) { + buffer[i] = counter++; + if (counter > 15) + counter = 0; + } + return 1; +} + +int test_import_small_file() { + size_t bytes_size = 1000; + unsigned char file_bytes[bytes_size]; + const char* fileName = "/tmp/test_import_small.tmp"; + + // create the necessary file + create_bytes(file_bytes, bytes_size); + create_file(fileName, file_bytes, bytes_size); + + // get the repo + drop_and_build_repository("/tmp/.ipfs"); + struct FSRepo* fs_repo; + ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, &fs_repo); + ipfs_repo_fsrepo_open(fs_repo); + + // write to ipfs + struct Node* write_node; + if (ipfs_import_file(fileName, &write_node, fs_repo) == 0) { + ipfs_repo_fsrepo_free(fs_repo); + return 0; + } + + // make sure all went okay + struct Node* read_node; + if (ipfs_merkledag_get(write_node->cached, &read_node, fs_repo) == 0) { + ipfs_repo_fsrepo_free(fs_repo); + ipfs_node_free(write_node); + return 0; + } + + // compare data + if (write_node->data_size != bytes_size || write_node->data_size != read_node->data_size) { + printf("Data size of nodes are not equal or are incorrect. Should be %lu but are %lu\n", write_node->data_size, read_node->data_size); + ipfs_repo_fsrepo_free(fs_repo); + ipfs_node_free(write_node); + ipfs_node_free(read_node); + return 0; + } + + for(int i = 0; i < bytes_size; i++) { + if (write_node->data[i] != read_node->data[i]) { + printf("Data within node is different at position %d\n", i); + ipfs_repo_fsrepo_free(fs_repo); + ipfs_node_free(write_node); + ipfs_node_free(read_node); + return 0; + } + } + + return 1; +} diff --git a/test/repo/test_repo_fsrepo.h b/test/repo/test_repo_fsrepo.h index 19c3bae..31b9c14 100644 --- a/test/repo/test_repo_fsrepo.h +++ b/test/repo/test_repo_fsrepo.h @@ -1,4 +1,5 @@ #include "ipfs/repo/fsrepo/fs_repo.h" +#include "../test_helper.h" int test_repo_fsrepo_open_config() { struct FSRepo* fs_repo = NULL; @@ -24,3 +25,73 @@ int test_repo_fsrepo_open_config() { return 1; } + +int test_repo_fsrepo_write_read_block() { + struct Block* block = NULL; + struct FSRepo* fs_repo = NULL; + int retVal = 0; + + // freshen the repository + retVal = drop_build_and_open_repo("/tmp/.ipfs", &fs_repo); + if (retVal == 0) + return 0; + + // make some data + size_t data_size = 100; + unsigned char data[data_size]; + + int counter = 0; + for(int i = 0; i < data_size; i++) { + data[i] = counter++; + if (counter > 15) + counter = 0; + } + + // create and write the block + retVal = ipfs_blocks_block_new(&block); + if (retVal == 0) { + ipfs_repo_fsrepo_free(fs_repo); + return 0; + } + retVal = ipfs_blocks_block_add_data(data, data_size, block); + if (retVal == 0) { + ipfs_repo_fsrepo_free(fs_repo); + return 0; + } + + retVal = ipfs_repo_fsrepo_block_write(block, fs_repo); + if (retVal == 0) { + ipfs_repo_fsrepo_free(fs_repo); + ipfs_blocks_block_free(block); + return 0; + } + + // retrieve the block + struct Block* results; + retVal = ipfs_repo_fsrepo_block_read(block->cid, &results, fs_repo); + if (retVal == 0) { + ipfs_repo_fsrepo_free(fs_repo); + ipfs_blocks_block_free(block); + return 0; + } + + // compare the two blocks + retVal = 1; + if (block->data_length != results->data_length || block->data_length != data_size) { + printf("block data is of different length: %lu vs %lu\n", results->data_length, block->data_length); + retVal = 0; + } + + for(size_t i = 0; i < block->data_length; i++) { + if (block->data[i] != results->data[i]) { + printf("Data is different at position %lu. Should be %02x but is %02x\n", i, block->data[i], results->data[i]); + retVal = 0; + break; + } + } + + ipfs_repo_fsrepo_free(fs_repo); + ipfs_blocks_block_free(block); + ipfs_blocks_block_free(results); + return retVal; +} diff --git a/test/storage/test_blocks.h b/test/storage/test_blocks.h index ada25b9..9a88e96 100644 --- a/test/storage/test_blocks.h +++ b/test/storage/test_blocks.h @@ -4,30 +4,48 @@ int test_blocks_new() { const unsigned char* input = (const unsigned char*)"Hello, World!"; int retVal = 0; struct Block* block; - retVal = ipfs_blocks_block_new(input, strlen((const char*)input) + 1, &block); + retVal = ipfs_blocks_block_new(&block); if (retVal == 0) return 0; + retVal = ipfs_blocks_block_add_data(input, strlen((const char*)input) + 1, block); + if (retVal == 0) { + ipfs_blocks_block_free(block); + return 0; + } + // now examine the block - if (strcmp((const char*)block->data, (const char*)input) != 0) + if (strcmp((const char*)block->data, (const char*)input) != 0) { + ipfs_blocks_block_free(block); return 0; + } - if (block->data_length != strlen((const char*)input) + 1) + if (block->data_length != strlen((const char*)input) + 1) { + ipfs_blocks_block_free(block); return 0; + } - if (block->cid->codec != CID_PROTOBUF) + if (block->cid->codec != CID_PROTOBUF) { + ipfs_blocks_block_free(block); return 0; + } - if (block->cid->version != 0) + if (block->cid->version != 0) { + ipfs_blocks_block_free(block); return 0; + } - if (block->cid->hash_length != 32) + if (block->cid->hash_length != 32) { + ipfs_blocks_block_free(block); return 0; + } unsigned char result_hash[32] = {33, 153, 66, 187, 124, 250, 87, 12, 12, 73, 43, 247, 175, 153, 10, 51, 192, 195, 218, 69, 220, 170, 105, 179, 195, 0, 203, 213, 172, 3, 244, 10 }; for(int i = 0; i < 32; i++) { - if (block->cid->hash[i] != result_hash[i]) + if (block->cid->hash[i] != result_hash[i]) { + ipfs_blocks_block_free(block); return 0; + } } retVal = ipfs_blocks_block_free(block); diff --git a/test/storage/test_datastore.h b/test/storage/test_datastore.h index c879a91..1152de4 100644 --- a/test/storage/test_datastore.h +++ b/test/storage/test_datastore.h @@ -23,30 +23,43 @@ int test_ipfs_datastore_put() { return 0; // build the block - retVal = ipfs_blocks_block_new(input, strlen((char*)input), &block); + retVal = ipfs_blocks_block_new(&block); if (retVal == 0) return 0; + retVal = ipfs_blocks_block_add_data(input, strlen((char*)input), block); + if (retVal == 0) { + ipfs_blocks_block_free(block); + return 0; + } + // generate the key size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length); unsigned char key[key_length]; retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length); - if (retVal == 0) + if (retVal == 0) { + ipfs_blocks_block_free(block); return 0; + } // open the repository struct FSRepo* fs_repo; retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, &fs_repo); - if (retVal == 0) + if (retVal == 0) { + ipfs_blocks_block_free(block); return 0; + } retVal = ipfs_repo_fsrepo_open(fs_repo); - if (retVal == 0) + if (retVal == 0) { + ipfs_blocks_block_free(block); return 0; - + } // send to Put with key retVal = fs_repo->config->datastore->datastore_put((const unsigned char*)key, key_length, block->data, block->data_length, fs_repo->config->datastore); - if (retVal == 0) + if (retVal == 0) { + ipfs_blocks_block_free(block); return 0; + } // save the block diff --git a/test/test_helper.c b/test/test_helper.c index 592669a..453e59b 100644 --- a/test/test_helper.c +++ b/test/test_helper.c @@ -122,3 +122,22 @@ int make_ipfs_repository(const char* path) { int drop_and_build_repository(const char* path) { return make_ipfs_repository(path); } + + +int drop_build_and_open_repo(const char* path, struct FSRepo** fs_repo) { + int retVal = 0; + + retVal = drop_and_build_repository("/tmp/.ipfs"); + if (retVal == 0) + return 0; + retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, fs_repo); + if (retVal == 0) + return 0; + retVal = ipfs_repo_fsrepo_open(*fs_repo); + if (retVal == 0) { + free(*fs_repo); + return 0; + } + return 1; +} + diff --git a/test/test_helper.h b/test/test_helper.h index 1d5b8ce..f46fbfd 100644 --- a/test/test_helper.h +++ b/test/test_helper.h @@ -1,5 +1,11 @@ +/** + * Helpers for testing + */ + /** * Create a new repository in the directory, erasing old one * NOTE: base directory must already exist */ int drop_and_build_repository(const char* dir); + +int drop_build_and_open_repo(const char* path, struct FSRepo** fs_repo); diff --git a/test/testit.c b/test/testit.c index a441c4b..1d6f02d 100644 --- a/test/testit.c +++ b/test/testit.c @@ -3,6 +3,7 @@ #include "flatfs/test_flatfs.h" #include "merkledag/test_merkledag.h" #include "node/test_node.h" +#include "node/test_importer.h" #include "repo/test_repo_bootstrap_peers.h" #include "repo/test_repo_config.h" #include "repo/test_repo_fsrepo.h" @@ -31,7 +32,9 @@ const char* names[] = { "test_repo_config_write", "test_repo_config_identity_new", "test_repo_config_identity_private_key", + "test_repo_fsrepo_write_read_block", "test_get_init_command", + "test_import_small_file", "test_repo_fsrepo_open_config", "test_flatfs_get_directory", "test_flatfs_get_filename", @@ -58,7 +61,9 @@ int (*funcs[])(void) = { test_repo_config_write, test_repo_config_identity_new, test_repo_config_identity_private_key, + test_repo_fsrepo_write_read_block, test_get_init_command, + test_import_small_file, test_repo_fsrepo_open_config, test_flatfs_get_directory, test_flatfs_get_filename,