From 861ca0a332054f4938942b139fe772e45c8b2a6c Mon Sep 17 00:00:00 2001 From: John Jones Date: Thu, 5 Oct 2017 13:08:36 -0500 Subject: [PATCH] Conversion from blockstore to node was not happening --- blocks/blockstore.c | 12 ++++- importer/exporter.c | 6 +-- include/ipfs/importer/exporter.h | 3 +- main/main.c | 2 +- routing/online.c | 2 +- test/core/test_api.h | 93 +++++++++++++++++++++++++++++++- test/scripts/test_1.sh | 8 +-- test/testit.c | 2 + 8 files changed, 116 insertions(+), 12 deletions(-) diff --git a/blocks/blockstore.c b/blocks/blockstore.c index a53836f..c5a0ecb 100644 --- a/blocks/blockstore.c +++ b/blocks/blockstore.c @@ -343,10 +343,20 @@ int ipfs_blockstore_get_node(const unsigned char* hash, size_t hash_length, stru size_t bytes_read = fread(buffer, 1, file_size, file); fclose(file); - int retVal = ipfs_hashtable_node_protobuf_decode(buffer, bytes_read, node); + // now we have the block, convert it to a node + struct Block* block; + if (!ipfs_blocks_block_protobuf_decode(buffer, bytes_read, &block)) { + free(key); + free(filename); + ipfs_block_free(block); + return 0; + } + + int retVal = ipfs_hashtable_node_protobuf_decode(block->data, block->data_length, node); free(key); free(filename); + ipfs_block_free(block); return retVal; } diff --git a/importer/exporter.c b/importer/exporter.c index 4a3b6fe..88acc77 100644 --- a/importer/exporter.c +++ b/importer/exporter.c @@ -267,7 +267,7 @@ int ipfs_exporter_object_cat_to_file(struct IpfsNode *local_node, unsigned char* * @param argv arguments * @returns true(1) on success */ -int ipfs_exporter_object_cat(struct CliArguments* args) { +int ipfs_exporter_object_cat(struct CliArguments* args, FILE* output_file) { struct IpfsNode *local_node = NULL; char* repo_dir = NULL; @@ -293,7 +293,7 @@ int ipfs_exporter_object_cat(struct CliArguments* args) { size_t response_size = 0; int retVal = ipfs_core_http_request_get(local_node, request, &response, &response_size); if (response != NULL && response_size > 0) { - fwrite(response, 1, response_size, stdout); + fwrite(response, 1, response_size, output_file); free(response); } else { retVal = 0; @@ -308,7 +308,7 @@ int ipfs_exporter_object_cat(struct CliArguments* args) { return 0; } - int retVal = ipfs_exporter_object_cat_to_file(local_node, cid->hash, cid->hash_length, stdout); + int retVal = ipfs_exporter_object_cat_to_file(local_node, cid->hash, cid->hash_length, output_file); ipfs_cid_free(cid); return retVal; diff --git a/include/ipfs/importer/exporter.h b/include/ipfs/importer/exporter.h index 3827c66..39e6ffc 100644 --- a/include/ipfs/importer/exporter.h +++ b/include/ipfs/importer/exporter.h @@ -31,9 +31,10 @@ int ipfs_exporter_object_get(int argc, char** argv); * Called from the command line with ipfs cat [hash]. Retrieves the object pointed to by hash, and displays its block data (links and data elements) * @param argc number of arguments * @param argv arguments + * @param output_file where to stream the results * @returns true(1) on success */ -int ipfs_exporter_object_cat(struct CliArguments* args); +int ipfs_exporter_object_cat(struct CliArguments* args, FILE* output_file); /** * Retrieves the object pointed to by hash and displays the raw data diff --git a/main/main.c b/main/main.c index 3e416ad..b85302a 100644 --- a/main/main.c +++ b/main/main.c @@ -171,7 +171,7 @@ int main(int argc, char** argv) { //ipfs_exporter_get(argc, argv); //break; case (CAT): - retVal = ipfs_exporter_object_cat(args); + retVal = ipfs_exporter_object_cat(args, stdout); break; case (DNS): retVal = ipfs_dns(argc, argv); diff --git a/routing/online.c b/routing/online.c index 3b1fe91..1f507bb 100644 --- a/routing/online.c +++ b/routing/online.c @@ -390,7 +390,7 @@ int ipfs_routing_online_get_value (ipfs_routing* routing, const unsigned char *k if (current_peer->is_local) { // it's a local fetch. Retrieve it libp2p_logger_debug("online", "It is a local fetch. Attempting get_value locally.\n"); - if (ipfs_routing_generic_get_value(routing, key, key_size, buffer, buffer_size) == 0) { + if (ipfs_routing_generic_get_value(routing, key, key_size, buffer, buffer_size)) { retVal = 1; break; } diff --git a/test/core/test_api.h b/test/core/test_api.h index a960840..5f0d69b 100644 --- a/test/core/test_api.h +++ b/test/core/test_api.h @@ -128,7 +128,7 @@ int test_core_api_object_cat() { // use a client to ask for the file on server 1 arguments = cli_arguments_new(5, args); - if (ipfs_exporter_object_cat(arguments) == 0) { + if (ipfs_exporter_object_cat(arguments, stdout) == 0) { libp2p_logger_error("test_api", "ipfs_exporter_object_cat returned false.\n"); goto exit; } @@ -213,7 +213,7 @@ int test_core_api_object_cat_binary() { // use a client to ask for the file arguments = cli_arguments_new(5, args); - if (ipfs_exporter_object_cat(arguments) == 0) { + if (ipfs_exporter_object_cat(arguments, stdout) == 0) { libp2p_logger_error("test_api", "ipfs_exporter_object_cat returned false.\n"); goto exit; } @@ -227,6 +227,95 @@ int test_core_api_object_cat_binary() { return retVal; } +/*** + * Attempt to get a large (>256K) binary file over the api + */ +int test_core_api_object_cat_large_binary() { + int retVal = 0; + pthread_t daemon_thread1; + int thread_started1 = 0; + char* ipfs_path1 = "/tmp/ipfs_1"; + char* config_file1 = "config.test1.wo_journal"; + struct FSRepo* fs_repo = NULL; + char hash[256] = ""; + char* args[] = {"ipfs", "--config", ipfs_path1, "cat", hash }; + struct CliArguments* arguments = NULL; + + // logging + libp2p_logger_add_class("test_api"); + libp2p_logger_add_class("journal"); + libp2p_logger_add_class("daemon"); + libp2p_logger_add_class("online"); + libp2p_logger_add_class("peer"); + libp2p_logger_add_class("null"); + libp2p_logger_add_class("replication"); + libp2p_logger_add_class("fs_repo"); + libp2p_logger_add_class("lmdb_journalstore"); + libp2p_logger_add_class("lmdb_datastore"); + libp2p_logger_add_class("secio"); + libp2p_logger_add_class("socket"); + libp2p_logger_add_class("protocol"); + libp2p_logger_add_class("dht_protocol"); + libp2p_logger_add_class("resolver"); + libp2p_logger_add_class("unixfs"); + libp2p_logger_add_class("bitswap_engine"); + libp2p_logger_add_class("bitswap_network"); + libp2p_logger_add_class("exporter"); + libp2p_logger_add_class("api"); + + // build repo + if (!drop_build_open_repo(ipfs_path1, &fs_repo, config_file1)) { + ipfs_repo_fsrepo_free(fs_repo); + libp2p_logger_error("test_api", "Unable to drop and build repository at %s\n", ipfs_path1); + goto exit; + } + libp2p_logger_debug("test_api", "Changed the server id to %s.\n", fs_repo->config->identity->peer->id); + ipfs_repo_fsrepo_free(fs_repo); + + // add a file to the repo + uint8_t bytes[300000]; + for(int i = 0; i < 300000; i++) + bytes[i] = i % 255; + char* filename = "test1.txt"; + create_file(filename, bytes, 300000); + struct HashtableNode* node; + size_t bytes_written; + struct IpfsNode *local_node = NULL; + ipfs_node_offline_new(ipfs_path1, &local_node); + ipfs_import_file(NULL, filename, &node, local_node, &bytes_written, 0); + memset(hash, 0, 256); + ipfs_cid_hash_to_base58(node->hash, node->hash_size, (unsigned char*)hash, 256); + libp2p_logger_debug("test_api", "Inserted file with hash %s.\n", hash); + ipfs_node_free(local_node); + ipfs_hashtable_node_free(node); + + libp2p_logger_debug("test_api", "*** Firing up daemons ***\n"); + pthread_create(&daemon_thread1, NULL, test_daemon_start, (void*)ipfs_path1); + thread_started1 = 1; + sleep(3); + + // use a client to ask for the file + arguments = cli_arguments_new(5, args); + char* filename2 = "test2.txt"; + unlink(filename2); + FILE* fd = fopen(filename2, "w" ); + if (ipfs_exporter_object_cat(arguments, fd) == 0) { + libp2p_logger_error("test_api", "ipfs_exporter_object_cat returned false.\n"); + fclose(fd); + goto exit; + } + fclose(fd); + + retVal = 1; + exit: + ipfs_daemon_stop(); + if (thread_started1) + pthread_join(daemon_thread1, NULL); + cli_arguments_free(arguments); + return retVal; +} + + int test_core_api_name_resolve() { int retVal = 0; pthread_t daemon_thread1; diff --git a/test/scripts/test_1.sh b/test/scripts/test_1.sh index 97d890a..1bb2789 100755 --- a/test/scripts/test_1.sh +++ b/test/scripts/test_1.sh @@ -23,7 +23,8 @@ function post { } function body { - create_binary_file 512; + retVal=0 + create_binary_file 512000; eval "$IPFS" add hello.bin check_failure_with_exit "add hello.bin" $? @@ -33,14 +34,15 @@ function body { sleep 5 eval "$IPFS" cat QmVGA3bXDJ41xoT621xQNzQgtBMQHj2AYxaunLBJtSoNgg > hello2.bin - check_failure_with_exit "cat" $? + check_failure "cat" $? # file size should be 512 actualsize=$(wc -c < hello2.bin) if [ $actualsize -ne 512 ]; then echo '*** Failure *** file size incorrect' - exit 1 + let retVal=1 fi kill -9 $daemon_id + exit $retVal } diff --git a/test/testit.c b/test/testit.c index 43fcddc..adbf650 100644 --- a/test/testit.c +++ b/test/testit.c @@ -54,6 +54,7 @@ const char* names[] = { "test_core_api_startup_shutdown", "test_core_api_object_cat", "test_core_api_object_cat_binary", + "test_core_api_object_cat_large_binary", "test_core_api_name_resolve", "test_core_api_name_resolve_1", "test_core_api_name_resolve_2", @@ -127,6 +128,7 @@ int (*funcs[])(void) = { test_core_api_startup_shutdown, test_core_api_object_cat, test_core_api_object_cat_binary, + test_core_api_object_cat_large_binary, test_core_api_name_resolve, test_core_api_name_resolve_1, test_core_api_name_resolve_2,