From 033dd767b4e767923959de1b33494dc3e4815e2c Mon Sep 17 00:00:00 2001 From: jmjatlanta Date: Wed, 14 Dec 2016 12:07:43 -0500 Subject: [PATCH] More work on persisting data to disk. Blockstore now storing the data, whereas datastore is storing the key and filename. The key should be the multihash (currently the sha256, not the multihash), and the value is the filename (base32). --- blocks/block.c | 117 ++++++++++++++++++++++++++--- blocks/blockstore.c | 98 +++++++++++++++++++++--- datastore/ds_helper.c | 5 +- importer/importer.c | 18 +++-- include/ipfs/blocks/block.h | 31 +++++++- include/ipfs/blocks/blockstore.h | 9 ++- include/ipfs/importer/importer.h | 3 +- include/ipfs/repo/fsrepo/fs_repo.h | 9 +++ merkledag/merkledag.c | 22 ++++-- node/node.c | 1 + os/utils.c | 5 +- repo/fsrepo/fs_repo.c | 63 +++++++++++++++- repo/fsrepo/lmdb_datastore.c | 5 +- test/node/test_importer.h | 77 +++++++++++++++++++ test/repo/test_repo_fsrepo.h | 71 +++++++++++++++++ test/storage/test_blocks.h | 32 ++++++-- test/storage/test_datastore.h | 25 ++++-- test/test_helper.c | 19 +++++ test/test_helper.h | 6 ++ test/testit.c | 5 ++ 20 files changed, 564 insertions(+), 57 deletions(-) create mode 100644 test/node/test_importer.h diff --git a/blocks/block.c b/blocks/block.c index 4d6dcf6..c917526 100644 --- a/blocks/block.c +++ b/blocks/block.c @@ -9,6 +9,100 @@ #include "ipfs/blocks/block.h" #include "ipfs/cid/cid.h" +/*** + * The protobuf functions + */ +// protobuf fields: data & data_length cid +enum WireType ipfs_block_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED}; + +/** + * Determine the approximate size of an encoded block + * @param block the block to measure + * @returns the approximate size needed to encode the protobuf + */ +size_t ipfs_blocks_block_protobuf_encode_size(const struct Block* block) { + return 22 + ipfs_cid_protobuf_encode_size(block->cid) + block->data_length; +} + +/** + * Encode the Block into protobuf format + * @param block the block to encode + * @param buffer the buffer to fill + * @param max_buffer_size the max size of the buffer + * @param bytes_written the number of bytes used + * @returns true(1) on success + */ +int ipfs_blocks_block_protobuf_encode(const struct Block* block, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) { + // data & data_size + size_t bytes_used = 0; + *bytes_written = 0; + int retVal = 0; + retVal = protobuf_encode_length_delimited(1, ipfs_block_message_fields[0], (char*)block->data, block->data_length, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + *bytes_written += bytes_used; + // cid + size_t cid_size = ipfs_cid_protobuf_encode_size(block->cid); + unsigned char cid[cid_size]; + retVal = ipfs_cid_protobuf_encode(block->cid, cid, cid_size, &cid_size); + retVal = protobuf_encode_length_delimited(2, ipfs_block_message_fields[1], (char*)cid, cid_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + *bytes_written += bytes_used; + return 1; +} + +/*** + * Decode from a protobuf stream into a Block struct + * @param buffer the buffer to pull from + * @param buffer_length the length of the buffer + * @param block the block to fill + * @returns true(1) on success + */ +int ipfs_blocks_block_protobuf_decode(const unsigned char* buffer, const size_t buffer_length, struct Block** block) { + size_t pos = 0; + int retVal = 0; + unsigned char* temp_buffer = NULL; + size_t temp_size; + + if (ipfs_blocks_block_new(block) == 0) + goto exit; + + while(pos < buffer_length) { + size_t bytes_read = 0; + int field_no; + enum WireType field_type; + if (protobuf_decode_field_and_type(&buffer[pos], buffer_length, &field_no, &field_type, &bytes_read) == 0) { + goto exit; + } + pos += bytes_read; + switch(field_no) { + case (1): // data + if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*block)->data), &((*block)->data_length), &bytes_read) == 0) + goto exit; + pos += bytes_read; + break; + case (2): // cid + if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0) + goto exit; + pos += bytes_read; + if (ipfs_cid_protobuf_decode(temp_buffer, temp_size, &((*block)->cid)) == 0) + goto exit; + free(temp_buffer); + temp_buffer = NULL; + break; + } + } + + retVal = 1; + +exit: + if (retVal == 0) { + ipfs_blocks_block_free(*block); + } + if (temp_buffer != NULL) + free(temp_buffer); + + return retVal; +} + + /*** * Create a new block based on the incoming data * @param data the data to base the block on @@ -16,35 +110,38 @@ * @param block a pointer to the struct Block that will be created * @returns true(1) on success */ -int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block) { +int ipfs_blocks_block_new(struct Block** block) { // allocate memory for structure (*block) = (struct Block*)malloc(sizeof(struct Block)); if ((*block) == NULL) return 0; + (*block)->data = NULL; + (*block)->data_length = 0; + return 1; +} + +int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, struct Block* block) { // cid unsigned char hash[32]; if (libp2p_crypto_hashing_sha256(data, data_size, &hash[0]) == 0) { - free(*block); return 0; } - if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &((*block)->cid)) == 0) { - free(*block); + if (ipfs_cid_new(0, hash, 32, CID_PROTOBUF, &(block->cid)) == 0) { return 0; } - (*block)->data_length = data_size; + block->data_length = data_size; - (*block)->data = malloc(sizeof(unsigned char) * data_size); - if ( (*block)->data == NULL) { - ipfs_cid_free((*block)->cid); - free(*block); + block->data = malloc(sizeof(unsigned char) * data_size); + if ( block->data == NULL) { + ipfs_cid_free(block->cid); return 0; } - memcpy( (*block)->data, data, data_size); + memcpy( block->data, data, data_size); return 1; } diff --git a/blocks/blockstore.c b/blocks/blockstore.c index d9ffda8..99823ae 100644 --- a/blocks/blockstore.c +++ b/blocks/blockstore.c @@ -6,6 +6,7 @@ #include "ipfs/blocks/block.h" #include "ipfs/datastore/ds_helper.h" #include "ipfs/repo/fsrepo/fs_repo.h" +#include "ipfs/os/utils.h" /** * Delete a block based on its Cid @@ -25,14 +26,59 @@ int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo) { return 0; } +unsigned char* ipfs_blockstore_cid_to_base32(const struct Cid* cid) { + size_t key_length = libp2p_crypto_encoding_base32_encode_size(cid->hash_length); + unsigned char* buffer = (unsigned char*)malloc(key_length + 1); + if (buffer == NULL) + return NULL; + int retVal = ipfs_datastore_helper_ds_key_from_binary(cid->hash, cid->hash_length, &buffer[0], key_length, &key_length); + if (retVal == 0) { + free(buffer); + return NULL; + } + buffer[key_length] = 0; + return buffer; +} + +char* ipfs_blockstore_path_get(const struct FSRepo* fs_repo, const char* filename) { + int filepath_size = strlen(fs_repo->path) + 12; + char filepath[filepath_size]; + int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", filepath, filepath_size); + if (retVal == 0) { + free(filepath); + return 0; + } + int complete_filename_size = strlen(filepath) + strlen(filename) + 2; + char* complete_filename = (char*)malloc(complete_filename_size); + retVal = os_utils_filepath_join(filepath, filename, complete_filename, complete_filename_size); + return complete_filename; +} + /*** * Find a block based on its Cid * @param cid the Cid to look for * @param block where to put the data to be returned * @returns true(1) on success */ -int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo) { - return 0; +int ipfs_blockstore_get(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo) { + // get datastore key, which is a base32 key of the multihash + unsigned char* key = ipfs_blockstore_cid_to_base32(cid); + + char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key); + + size_t file_size = os_utils_file_size(filename); + unsigned char buffer[file_size]; + + FILE* file = fopen(filename, "rb"); + size_t bytes_read = fread(buffer, 1, file_size, file); + fclose(file); + + int retVal = ipfs_blocks_block_protobuf_decode(buffer, bytes_read, block); + + free(key); + free(filename); + + return retVal; } /*** @@ -42,14 +88,46 @@ int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_ */ int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) { // from blockstore.go line 118 - // Get Datastore key, which is a base32 key of the binary, - size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length); - unsigned char key[key_length]; - int retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length); - if (retVal == 0) - return 0; + int retVal = 0; - // send to Put with key - fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore); + // Get Datastore key, which is a base32 key of the multihash, + unsigned char* key = ipfs_blockstore_cid_to_base32(block->cid); + if (key == NULL) { + free(key); + return 0; + } + + //TODO: put this in subdirectories + + // turn the block into a binary array + size_t protobuf_len = ipfs_blocks_block_protobuf_encode_size(block); + unsigned char protobuf[protobuf_len]; + retVal = ipfs_blocks_block_protobuf_encode(block, protobuf, protobuf_len, &protobuf_len); + if (retVal == 0) { + free(key); + return 0; + } + + // now write byte array to file + char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key); + if (filename == NULL) { + free(key); + return 0; + } + + FILE* file = fopen(filename, "wb"); + int bytes_written = fwrite(protobuf, 1, protobuf_len, file); + fclose(file); + if (bytes_written != protobuf_len) { + free(key); + free(filename); + return 0; + } + + // send to Put with key (this is now done separately) + //fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore); + + free(key); + free(filename); return 1; } diff --git a/datastore/ds_helper.c b/datastore/ds_helper.c index 2e1a75d..c78f483 100644 --- a/datastore/ds_helper.c +++ b/datastore/ds_helper.c @@ -1,10 +1,11 @@ /** * Some code to help with the datastore / blockstore interface + * NOTE: the datastore stores things under a multihash key */ #include "libp2p/crypto/encoding/base32.h" #include "ipfs/datastore/ds_helper.h" /** - * Generate a key based on the passed in binary_array + * Generate a base32 key based on the passed in binary_array (which is normally a multihash) * @param binary_array what to base the key on * @param array_length the size of the binary array * @param results where the key will be put @@ -30,7 +31,7 @@ int ipfs_datastore_helper_ds_key_from_binary(unsigned char* binary_array, size_t } /** - * Generate a binary array based on the passed in datastore key + * Generate a binary array (normally a multihash) based on the passed in datastore key * @param ds_key the base32 encoded key * @param key_length the length of the base32 "string" * @param binary_array where to put the decoded value diff --git a/importer/importer.c b/importer/importer.c index 585185f..524a803 100644 --- a/importer/importer.c +++ b/importer/importer.c @@ -1,6 +1,7 @@ #include #include "ipfs/importer/importer.h" +#include "ipfs/merkledag/merkledag.h" #define MAX_DATA_SIZE 262144 // 1024 * 256; @@ -14,26 +15,27 @@ * @param node the node to add to * @returns number of bytes read */ -size_t ipfs_import_chunk(FILE* file, struct Node* node) { +size_t ipfs_import_chunk(FILE* file, struct Node* node, struct FSRepo* fs_repo) { unsigned char buffer[MAX_DATA_SIZE]; - size_t bytes_read = fread(buffer, MAX_DATA_SIZE, 1, file); + size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file); if (node->data_size == 0) { ipfs_node_set_data(node, buffer, bytes_read); - if (bytes_read != MAX_DATA_SIZE) { - // persist - } } else { // create a new node, and link to the parent struct Node* new_node = NULL; ipfs_node_new_from_data(buffer, bytes_read, &new_node); // persist - + ipfs_merkledag_add(new_node, fs_repo); // put link in node struct NodeLink* new_link = NULL; ipfs_node_link_new("", new_node->cached->hash, &new_link); ipfs_node_add_link(node, new_link); ipfs_node_free(new_node); } + if (bytes_read != MAX_DATA_SIZE) { + // persist the main node + ipfs_merkledag_add(node, fs_repo); + } return bytes_read; } @@ -43,7 +45,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* node) { * @param node the root node (could have links to others) * @returns true(1) on success */ -int ipfs_import_file(const char* fileName, struct Node** node) { +int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo) { int retVal = 1; int bytes_read = MAX_DATA_SIZE; @@ -54,7 +56,7 @@ int ipfs_import_file(const char* fileName, struct Node** node) { // add all nodes while ( bytes_read == MAX_DATA_SIZE) { - bytes_read = ipfs_import_chunk(file, *node); + bytes_read = ipfs_import_chunk(file, *node, fs_repo); } fclose(file); diff --git a/include/ipfs/blocks/block.h b/include/ipfs/blocks/block.h index 288c3f0..9d27cdb 100644 --- a/include/ipfs/blocks/block.h +++ b/include/ipfs/blocks/block.h @@ -1,5 +1,6 @@ /*** * IPFS has the notion of storage blocks. + * Raw data with a multihash key (the Cid) */ #ifndef __IPFS_BLOCKS_BLOCK_H__ @@ -20,7 +21,9 @@ struct Block { * @param block a pointer to the struct Block that will be created * @returns true(1) on success */ -int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Block** block); +int ipfs_blocks_block_new(struct Block** block); + +int ipfs_blocks_block_add_data(const unsigned char* data, size_t data_size, struct Block* block); /*** * Free resources used by the creation of a block @@ -29,4 +32,30 @@ int ipfs_blocks_block_new(const unsigned char* data, size_t data_size, struct Bl */ int ipfs_blocks_block_free(struct Block* block); +/** + * Determine the approximate size of an encoded block + * @param block the block to measure + * @returns the approximate size needed to encode the protobuf + */ +size_t ipfs_blocks_block_protobuf_encode_size(const struct Block* block); + +/** + * Encode the Block into protobuf format + * @param block the block to encode + * @param buffer the buffer to fill + * @param max_buffer_size the max size of the buffer + * @param bytes_written the number of bytes used + * @returns true(1) on success + */ +int ipfs_blocks_block_protobuf_encode(const struct Block* block, unsigned char* buffer, size_t max_buffer_size, size_t* bytes_written); + +/*** + * Decode from a protobuf stream into a Block struct + * @param buffer the buffer to pull from + * @param buffer_length the length of the buffer + * @param block the block to fill + * @returns true(1) on success + */ +int ipfs_blocks_block_protobuf_decode(const unsigned char* buffer, const size_t buffer_length, struct Block** block); + #endif diff --git a/include/ipfs/blocks/blockstore.h b/include/ipfs/blocks/blockstore.h index f102c28..37f2569 100644 --- a/include/ipfs/blocks/blockstore.h +++ b/include/ipfs/blocks/blockstore.h @@ -3,7 +3,10 @@ */ #ifndef __IPFS_BLOCKS_BLOCKSTORE_H__ -#ifndef __IPFS_BLOCKS_BLOCKSTORE_H__ +#define __IPFS_BLOCKS_BLOCKSTORE_H__ + +#include "ipfs/cid/cid.h" +#include "ipfs/repo/fsrepo/fs_repo.h" /** * Delete a block based on its Cid @@ -25,14 +28,14 @@ int ipfs_blockstore_has(struct Cid* cid, struct FSRepo* fs_repo); * @param block where to put the data to be returned * @returns true(1) on success */ -int ipfs_blockstore_get(struct Cid* cid, struct Block* block, struct FSRepo* fs_repo); +int ipfs_blockstore_get(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo); /*** * Put a block in the blockstore * @param block the block to store * @returns true(1) on success */ -int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo); +int ipfs_blockstore_put(struct Block* block, const struct FSRepo* fs_repo); #endif diff --git a/include/ipfs/importer/importer.h b/include/ipfs/importer/importer.h index 01e44d3..b1292a7 100644 --- a/include/ipfs/importer/importer.h +++ b/include/ipfs/importer/importer.h @@ -2,6 +2,7 @@ #define __IPFS_IMPORTER_IMPORTER_H__ #include "ipfs/node/node.h" +#include "ipfs/repo/fsrepo/fs_repo.h" /** * Creates a node based on an incoming file @@ -9,6 +10,6 @@ * @param node the root node (could have links to others) * @returns true(1) on success */ -int ipfs_import_file(const char* fileName, struct Node** node); +int ipfs_import_file(const char* fileName, struct Node** node, struct FSRepo* fs_repo); #endif /* INCLUDE_IPFS_IMPORTER_IMPORTER_H_ */ diff --git a/include/ipfs/repo/fsrepo/fs_repo.h b/include/ipfs/repo/fsrepo/fs_repo.h index c866ce2..7ab6d2c 100644 --- a/include/ipfs/repo/fsrepo/fs_repo.h +++ b/include/ipfs/repo/fsrepo/fs_repo.h @@ -64,4 +64,13 @@ int ipfs_repo_fsrepo_free(struct FSRepo* config); */ int ipfs_repo_fsrepo_init(struct FSRepo* config); +/*** + * Write a block to the datastore and blockstore + * @param block the block to write + * @param fs_repo the repo to write to + * @returns true(1) on success + */ +int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo); +int ipfs_repo_fsrepo_block_read(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo); + #endif /* fs_repo_h */ diff --git a/merkledag/merkledag.c b/merkledag/merkledag.c index e9a0bc4..0753fd2 100644 --- a/merkledag/merkledag.c +++ b/merkledag/merkledag.c @@ -21,9 +21,11 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) { // turn the node into a block struct Block* block; - ipfs_blocks_block_new(protobuf, bytes_written, &block); + ipfs_blocks_block_new(&block); + ipfs_blocks_block_add_data(protobuf, bytes_written, block); - int retVal = fs_repo->config->datastore->datastore_put_block(block, fs_repo->config->datastore); + // write to block store & datastore + int retVal = ipfs_repo_fsrepo_block_write(block, fs_repo); if (retVal == 0) { ipfs_blocks_block_free(block); return 0; @@ -45,17 +47,25 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) { */ int ipfs_merkledag_get(const struct Cid* cid, struct Node** node, const struct FSRepo* fs_repo) { int retVal = 1; - - // look for the block struct Block* block; - retVal = fs_repo->config->datastore->datastore_get_block(cid, &block, fs_repo->config->datastore); + size_t key_length = 100; + unsigned char key[key_length]; + + // look for the node in the datastore. If it is not there, it is not a node. + // If it exists, it is only a block. + retVal = fs_repo->config->datastore->datastore_get((char*)cid->hash, cid->hash_length, key, key_length, &key_length, fs_repo->config->datastore); if (retVal == 0) return 0; - // we have the block. Fill the node + // we have the record from the db. Go get the block from the blockstore + retVal = ipfs_repo_fsrepo_block_read(cid, &block, fs_repo); + + // now convert the block into a node ipfs_node_protobuf_decode(block->data, block->data_length, node); + // doesn't decode do this? ipfs_node_set_cached(*node, cid); + // free resources ipfs_blocks_block_free(block); return retVal; diff --git a/node/node.c b/node/node.c index 8de07b8..ac964d7 100644 --- a/node/node.c +++ b/node/node.c @@ -295,6 +295,7 @@ int ipfs_node_new(struct Node** node) return 0; (*node)->cached = NULL; (*node)->data = NULL; + (*node)->data_size = 0; (*node)->encoded = NULL; (*node)->head_link = NULL; return 1; diff --git a/os/utils.c b/os/utils.c index e35f097..41dc01c 100644 --- a/os/utils.c +++ b/os/utils.c @@ -62,11 +62,14 @@ int os_utils_directory_writeable(const char* path) { } int os_utils_file_size(const char* path) { + size_t file_size = 0; // open file FILE* in_file = fopen(path, "r"); + if (in_file == NULL) + return 0; // determine size fseek(in_file, 0L, SEEK_END); - size_t file_size = ftell(in_file); + file_size = ftell(in_file); fclose(in_file); return file_size; } diff --git a/repo/fsrepo/fs_repo.c b/repo/fsrepo/fs_repo.c index 5117ab6..0d8ae7c 100644 --- a/repo/fsrepo/fs_repo.c +++ b/repo/fsrepo/fs_repo.c @@ -1,6 +1,9 @@ #include +#include +#include "ipfs/blocks/blockstore.h" #include "libp2p/crypto/encoding/base64.h" +#include "ipfs/datastore/ds_helper.h" #include "ipfs/repo/config/datastore.h" #include "ipfs/repo/fsrepo/fs_repo.h" #include "ipfs/os/utils.h" @@ -460,6 +463,18 @@ int ipfs_repo_fsrepo_datastore_init(struct FSRepo* fs_repo) { return repo_fsrepo_lmdb_cast(fs_repo->config->datastore); } +int ipfs_repo_fsrepo_blockstore_init(const struct FSRepo* fs_repo) { + size_t full_path_size = strlen(fs_repo->path) + 15; + char full_path[full_path_size]; + int retVal = os_utils_filepath_join(fs_repo->path, "blockstore", full_path, full_path_size); + if (retVal == 0) + return 0; + + if (mkdir(full_path, S_IRWXU) != 0) + return 0; + return 1; +} + /** * Initializes a new FSRepo at the given path with the provided config * @param path the path to use @@ -477,11 +492,14 @@ int ipfs_repo_fsrepo_init(struct FSRepo* repo) { if (retVal == 0) return 0; - // TODO: Implement this method retVal = ipfs_repo_fsrepo_datastore_init(repo); if (retVal == 0) return 0; + retVal = ipfs_repo_fsrepo_blockstore_init(repo); + if (retVal == 0) + return 0; + // write the version to a file for migrations (see repo/fsrepo/migrations/mfsr.go) //TODO: mfsr.RepoPath(repo_path).WriteVersion(RepoVersion) return 1; @@ -508,3 +526,46 @@ int fs_repo_write_config_file(char* path, struct RepoConfig* config) { return retVal; } +/*** + * Write a block to the datastore and blockstore + * @param block the block to write + * @param fs_repo the repo to write to + * @returns true(1) on success + */ +int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo) { + /** + * What is put in the blockstore is the block. + * What is put in the datastore is the multihash (the Cid) as the key, + * and the base32 encoded multihash as the value. + */ + int retVal = 1; + retVal = ipfs_blockstore_put(block, fs_repo); + if (retVal == 0) + return 0; + // take the cid, base32 it, and send both to the datastore + size_t fs_key_length = 100; + unsigned char fs_key[fs_key_length]; + retVal = ipfs_datastore_helper_ds_key_from_binary(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, &fs_key_length); + if (retVal == 0) + return 0; + retVal = fs_repo->config->datastore->datastore_put(block->cid->hash, block->cid->hash_length, fs_key, fs_key_length, fs_repo->config->datastore); + if (retVal == 0) + return 0; + return 1; +} + +int ipfs_repo_fsrepo_block_read(const struct Cid* cid, struct Block** block, const struct FSRepo* fs_repo) { + int retVal = 0; + + // get the base32 hash from the database + // We do this only to see if it is in the database + size_t fs_key_length = 100; + unsigned char fs_key[fs_key_length]; + retVal = fs_repo->config->datastore->datastore_get((char*)cid->hash, cid->hash_length, fs_key, fs_key_length, &fs_key_length, fs_repo->config->datastore); + if (retVal == 0) // maybe it doesn't exist? + return 0; + // now get the block from the blockstore + retVal = ipfs_blockstore_get(cid, block, fs_repo); + return retVal; +} + diff --git a/repo/fsrepo/lmdb_datastore.c b/repo/fsrepo/lmdb_datastore.c index 1d632e0..9fc25de 100644 --- a/repo/fsrepo/lmdb_datastore.c +++ b/repo/fsrepo/lmdb_datastore.c @@ -1,5 +1,7 @@ /*** * Here are the wrappers for the lightning database + * NOTE: In this implementation, the database will contain the base32 encoded value + * of the multihash key if the file exists on disk. */ #include @@ -43,12 +45,13 @@ int repo_fsrepo_lmdb_get_block(const struct Cid* cid, struct Block** block, cons } // now copy the data - retVal = ipfs_blocks_block_new(db_value.mv_data, db_value.mv_size, block); + retVal = ipfs_blocks_block_new(block); if (retVal == 0) { mdb_dbi_close(mdb_env, mdb_dbi); mdb_txn_commit(mdb_txn); return 0; } + retVal = ipfs_blocks_block_add_data(db_value.mv_data, db_value.mv_size, *block); // clean up mdb_dbi_close(mdb_env, mdb_dbi); diff --git a/test/node/test_importer.h b/test/node/test_importer.h new file mode 100644 index 0000000..754a1c2 --- /dev/null +++ b/test/node/test_importer.h @@ -0,0 +1,77 @@ +#include + +#include "ipfs/importer/importer.h" +#include "ipfs/merkledag/merkledag.h" + +/*** + * Helper to create a test file in the OS + */ +int create_file(const char* fileName, unsigned char* bytes, size_t num_bytes) { + FILE* file = fopen(fileName, "wb"); + fwrite(bytes, num_bytes, 1, file); + fclose(file); + return 1; +} + +int create_bytes(unsigned char* buffer, size_t num_bytes) { + int counter = 0; + + for(int i = 0; i < num_bytes; i++) { + buffer[i] = counter++; + if (counter > 15) + counter = 0; + } + return 1; +} + +int test_import_small_file() { + size_t bytes_size = 1000; + unsigned char file_bytes[bytes_size]; + const char* fileName = "/tmp/test_import_small.tmp"; + + // create the necessary file + create_bytes(file_bytes, bytes_size); + create_file(fileName, file_bytes, bytes_size); + + // get the repo + drop_and_build_repository("/tmp/.ipfs"); + struct FSRepo* fs_repo; + ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, &fs_repo); + ipfs_repo_fsrepo_open(fs_repo); + + // write to ipfs + struct Node* write_node; + if (ipfs_import_file(fileName, &write_node, fs_repo) == 0) { + ipfs_repo_fsrepo_free(fs_repo); + return 0; + } + + // make sure all went okay + struct Node* read_node; + if (ipfs_merkledag_get(write_node->cached, &read_node, fs_repo) == 0) { + ipfs_repo_fsrepo_free(fs_repo); + ipfs_node_free(write_node); + return 0; + } + + // compare data + if (write_node->data_size != bytes_size || write_node->data_size != read_node->data_size) { + printf("Data size of nodes are not equal or are incorrect. Should be %lu but are %lu\n", write_node->data_size, read_node->data_size); + ipfs_repo_fsrepo_free(fs_repo); + ipfs_node_free(write_node); + ipfs_node_free(read_node); + return 0; + } + + for(int i = 0; i < bytes_size; i++) { + if (write_node->data[i] != read_node->data[i]) { + printf("Data within node is different at position %d\n", i); + ipfs_repo_fsrepo_free(fs_repo); + ipfs_node_free(write_node); + ipfs_node_free(read_node); + return 0; + } + } + + return 1; +} diff --git a/test/repo/test_repo_fsrepo.h b/test/repo/test_repo_fsrepo.h index 19c3bae..31b9c14 100644 --- a/test/repo/test_repo_fsrepo.h +++ b/test/repo/test_repo_fsrepo.h @@ -1,4 +1,5 @@ #include "ipfs/repo/fsrepo/fs_repo.h" +#include "../test_helper.h" int test_repo_fsrepo_open_config() { struct FSRepo* fs_repo = NULL; @@ -24,3 +25,73 @@ int test_repo_fsrepo_open_config() { return 1; } + +int test_repo_fsrepo_write_read_block() { + struct Block* block = NULL; + struct FSRepo* fs_repo = NULL; + int retVal = 0; + + // freshen the repository + retVal = drop_build_and_open_repo("/tmp/.ipfs", &fs_repo); + if (retVal == 0) + return 0; + + // make some data + size_t data_size = 100; + unsigned char data[data_size]; + + int counter = 0; + for(int i = 0; i < data_size; i++) { + data[i] = counter++; + if (counter > 15) + counter = 0; + } + + // create and write the block + retVal = ipfs_blocks_block_new(&block); + if (retVal == 0) { + ipfs_repo_fsrepo_free(fs_repo); + return 0; + } + retVal = ipfs_blocks_block_add_data(data, data_size, block); + if (retVal == 0) { + ipfs_repo_fsrepo_free(fs_repo); + return 0; + } + + retVal = ipfs_repo_fsrepo_block_write(block, fs_repo); + if (retVal == 0) { + ipfs_repo_fsrepo_free(fs_repo); + ipfs_blocks_block_free(block); + return 0; + } + + // retrieve the block + struct Block* results; + retVal = ipfs_repo_fsrepo_block_read(block->cid, &results, fs_repo); + if (retVal == 0) { + ipfs_repo_fsrepo_free(fs_repo); + ipfs_blocks_block_free(block); + return 0; + } + + // compare the two blocks + retVal = 1; + if (block->data_length != results->data_length || block->data_length != data_size) { + printf("block data is of different length: %lu vs %lu\n", results->data_length, block->data_length); + retVal = 0; + } + + for(size_t i = 0; i < block->data_length; i++) { + if (block->data[i] != results->data[i]) { + printf("Data is different at position %lu. Should be %02x but is %02x\n", i, block->data[i], results->data[i]); + retVal = 0; + break; + } + } + + ipfs_repo_fsrepo_free(fs_repo); + ipfs_blocks_block_free(block); + ipfs_blocks_block_free(results); + return retVal; +} diff --git a/test/storage/test_blocks.h b/test/storage/test_blocks.h index ada25b9..9a88e96 100644 --- a/test/storage/test_blocks.h +++ b/test/storage/test_blocks.h @@ -4,30 +4,48 @@ int test_blocks_new() { const unsigned char* input = (const unsigned char*)"Hello, World!"; int retVal = 0; struct Block* block; - retVal = ipfs_blocks_block_new(input, strlen((const char*)input) + 1, &block); + retVal = ipfs_blocks_block_new(&block); if (retVal == 0) return 0; + retVal = ipfs_blocks_block_add_data(input, strlen((const char*)input) + 1, block); + if (retVal == 0) { + ipfs_blocks_block_free(block); + return 0; + } + // now examine the block - if (strcmp((const char*)block->data, (const char*)input) != 0) + if (strcmp((const char*)block->data, (const char*)input) != 0) { + ipfs_blocks_block_free(block); return 0; + } - if (block->data_length != strlen((const char*)input) + 1) + if (block->data_length != strlen((const char*)input) + 1) { + ipfs_blocks_block_free(block); return 0; + } - if (block->cid->codec != CID_PROTOBUF) + if (block->cid->codec != CID_PROTOBUF) { + ipfs_blocks_block_free(block); return 0; + } - if (block->cid->version != 0) + if (block->cid->version != 0) { + ipfs_blocks_block_free(block); return 0; + } - if (block->cid->hash_length != 32) + if (block->cid->hash_length != 32) { + ipfs_blocks_block_free(block); return 0; + } unsigned char result_hash[32] = {33, 153, 66, 187, 124, 250, 87, 12, 12, 73, 43, 247, 175, 153, 10, 51, 192, 195, 218, 69, 220, 170, 105, 179, 195, 0, 203, 213, 172, 3, 244, 10 }; for(int i = 0; i < 32; i++) { - if (block->cid->hash[i] != result_hash[i]) + if (block->cid->hash[i] != result_hash[i]) { + ipfs_blocks_block_free(block); return 0; + } } retVal = ipfs_blocks_block_free(block); diff --git a/test/storage/test_datastore.h b/test/storage/test_datastore.h index c879a91..1152de4 100644 --- a/test/storage/test_datastore.h +++ b/test/storage/test_datastore.h @@ -23,30 +23,43 @@ int test_ipfs_datastore_put() { return 0; // build the block - retVal = ipfs_blocks_block_new(input, strlen((char*)input), &block); + retVal = ipfs_blocks_block_new(&block); if (retVal == 0) return 0; + retVal = ipfs_blocks_block_add_data(input, strlen((char*)input), block); + if (retVal == 0) { + ipfs_blocks_block_free(block); + return 0; + } + // generate the key size_t key_length = libp2p_crypto_encoding_base32_encode_size(block->data_length); unsigned char key[key_length]; retVal = ipfs_datastore_helper_ds_key_from_binary(block->data, block->data_length, &key[0], key_length, &key_length); - if (retVal == 0) + if (retVal == 0) { + ipfs_blocks_block_free(block); return 0; + } // open the repository struct FSRepo* fs_repo; retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, &fs_repo); - if (retVal == 0) + if (retVal == 0) { + ipfs_blocks_block_free(block); return 0; + } retVal = ipfs_repo_fsrepo_open(fs_repo); - if (retVal == 0) + if (retVal == 0) { + ipfs_blocks_block_free(block); return 0; - + } // send to Put with key retVal = fs_repo->config->datastore->datastore_put((const unsigned char*)key, key_length, block->data, block->data_length, fs_repo->config->datastore); - if (retVal == 0) + if (retVal == 0) { + ipfs_blocks_block_free(block); return 0; + } // save the block diff --git a/test/test_helper.c b/test/test_helper.c index 592669a..453e59b 100644 --- a/test/test_helper.c +++ b/test/test_helper.c @@ -122,3 +122,22 @@ int make_ipfs_repository(const char* path) { int drop_and_build_repository(const char* path) { return make_ipfs_repository(path); } + + +int drop_build_and_open_repo(const char* path, struct FSRepo** fs_repo) { + int retVal = 0; + + retVal = drop_and_build_repository("/tmp/.ipfs"); + if (retVal == 0) + return 0; + retVal = ipfs_repo_fsrepo_new("/tmp/.ipfs", NULL, fs_repo); + if (retVal == 0) + return 0; + retVal = ipfs_repo_fsrepo_open(*fs_repo); + if (retVal == 0) { + free(*fs_repo); + return 0; + } + return 1; +} + diff --git a/test/test_helper.h b/test/test_helper.h index 1d5b8ce..f46fbfd 100644 --- a/test/test_helper.h +++ b/test/test_helper.h @@ -1,5 +1,11 @@ +/** + * Helpers for testing + */ + /** * Create a new repository in the directory, erasing old one * NOTE: base directory must already exist */ int drop_and_build_repository(const char* dir); + +int drop_build_and_open_repo(const char* path, struct FSRepo** fs_repo); diff --git a/test/testit.c b/test/testit.c index a441c4b..1d6f02d 100644 --- a/test/testit.c +++ b/test/testit.c @@ -3,6 +3,7 @@ #include "flatfs/test_flatfs.h" #include "merkledag/test_merkledag.h" #include "node/test_node.h" +#include "node/test_importer.h" #include "repo/test_repo_bootstrap_peers.h" #include "repo/test_repo_config.h" #include "repo/test_repo_fsrepo.h" @@ -31,7 +32,9 @@ const char* names[] = { "test_repo_config_write", "test_repo_config_identity_new", "test_repo_config_identity_private_key", + "test_repo_fsrepo_write_read_block", "test_get_init_command", + "test_import_small_file", "test_repo_fsrepo_open_config", "test_flatfs_get_directory", "test_flatfs_get_filename", @@ -58,7 +61,9 @@ int (*funcs[])(void) = { test_repo_config_write, test_repo_config_identity_new, test_repo_config_identity_private_key, + test_repo_fsrepo_write_read_block, test_get_init_command, + test_import_small_file, test_repo_fsrepo_open_config, test_flatfs_get_directory, test_flatfs_get_filename,