agorise
/
c-ipfs
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Better handling of locally stored files

yamux
John Jones 2017-05-11 07:04:54 -05:00
parent 3de4b757e4
commit def5331d4c
11 changed files with 126 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
core/bootstrap.c
View File

@ -9,9 +9,9 @@
/***
* Begin to connect to the swarm
*/
/*
void *ipfs_bootstrap_swarm(void* param) {
//TODO:
/*
struct IpfsNode* local_node = (struct IpfsNode*)param;
// read the config file and get the bootstrap peers
for(int i = 0; i < local_node->repo->config->peer_addresses.num_peers; i++) { // loop through the peers
@ -30,14 +30,15 @@ void *ipfs_bootstrap_swarm(void* param) {
} // we have a good peer ID
}
*/
return (void*)1;
}
*/
/***
* Announce to the network all of the files that I have in storage
* @param local_node the context
*/
/*
void ipfs_bootstrap_announce_files(struct IpfsNode* local_node) {
struct Datastore* db = local_node->repo->config->datastore;
if (!db->datastore_cursor_open(db))
@ -57,6 +58,7 @@ void ipfs_bootstrap_announce_files(struct IpfsNode* local_node) {
return;
}
*/
/***
* connect to the swarm
@ -65,6 +67,7 @@ void ipfs_bootstrap_announce_files(struct IpfsNode* local_node) {
* @param param the IpfsNode information
* @returns nothing useful
*/
/*
void *ipfs_bootstrap_routing(void* param) {
struct IpfsNode* local_node = (struct IpfsNode*)param;
local_node->routing = ipfs_routing_new_online(local_node, &local_node->identity->private_key, NULL);
@ -72,3 +75,4 @@ void *ipfs_bootstrap_routing(void* param) {
ipfs_bootstrap_announce_files(local_node);
return (void*)2;
}
*/

2
core/ipfs_node.c
View File

@ -38,7 +38,7 @@ int ipfs_node_online_new(const char* repo_path, struct IpfsNode** node) {
// fill in the node
local_node->repo = fs_repo;
local_node->identity = fs_repo->config->identity;
local_node->peerstore = libp2p_peerstore_new();
local_node->peerstore = libp2p_peerstore_new(local_node->identity->peer_id);
local_node->providerstore = libp2p_providerstore_new();
local_node->mode = MODE_OFFLINE;
local_node->routing = ipfs_routing_new_online(local_node, &fs_repo->config->identity->private_key, NULL);

2
core/ping.c
View File

@ -50,7 +50,7 @@ int ipfs_ping (int argc, char **argv)
local_node.repo = fs_repo;
local_node.mode = MODE_ONLINE;
local_node.routing = ipfs_routing_new_online(&local_node, &fs_repo->config->identity->private_key, stream);
local_node.peerstore = libp2p_peerstore_new();
local_node.peerstore = libp2p_peerstore_new(local_node.identity->peer_id);
local_node.providerstore = libp2p_providerstore_new();
if (local_node.routing->Bootstrap(local_node.routing) != 0)

4
importer/exporter.c
View File

@ -84,7 +84,6 @@ int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor
ipfs_unixfs_protobuf_decode(read_node->data, read_node->data_size, &unix_fs);
size_t bytes_written = fwrite(unix_fs->bytes, 1, unix_fs->bytes_size, file_descriptor);
if (bytes_written != unix_fs->bytes_size) {
fclose(file_descriptor);
ipfs_hashtable_node_free(read_node);
ipfs_unixfs_free(unix_fs);
return 0;
@ -95,7 +94,6 @@ int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor
struct HashtableNode* link_node = NULL;
while (link != NULL) {
if ( !ipfs_exporter_get_node(local_node, link->hash, link->hash_size, &link_node)) {
fclose(file_descriptor);
ipfs_hashtable_node_free(read_node);
return 0;
}
@ -104,7 +102,6 @@ int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor
size_t bytes_written = fwrite(unix_fs->bytes, 1, unix_fs->bytes_size, file_descriptor);
if (bytes_written != unix_fs->bytes_size) {
ipfs_hashtable_node_free(link_node);
fclose(file_descriptor);
ipfs_hashtable_node_free(read_node);
ipfs_unixfs_free(unix_fs);
return 0;
@ -114,7 +111,6 @@ int ipfs_exporter_to_filestream(const unsigned char* hash, FILE* file_descriptor
link = link->next;
}
}
fclose(file_descriptor);
if (read_node != NULL)
ipfs_hashtable_node_free(read_node);

30
routing/offline.c
View File

@ -39,19 +39,31 @@ int ipfs_routing_generic_put_value (ipfs_routing* offlineRouting, const unsigned
int ipfs_routing_generic_get_value (ipfs_routing* routing, const unsigned char *key, size_t key_size, void **val, size_t *vlen)
{
char key_str[key_size + 1];
strncpy(key_str, (const char*)key, key_size);
key_str[key_size] = 0;
struct HashtableNode* node = ipfs_resolver_get(key_str, NULL, routing->local_node);
if (node == NULL)
return -1;
struct HashtableNode* node = NULL;
*val = NULL;
int retVal = -1;
if (!ipfs_merkledag_get(key, key_size, &node, routing->local_node->repo)) {
goto exit;
}
// protobuf the node
int protobuf_size = ipfs_hashtable_node_protobuf_encode_size(node);
*val = malloc(protobuf_size);
if (ipfs_hashtable_node_protobuf_encode(node, *val, protobuf_size, vlen) == 0)
return -1;
return 0;
if (ipfs_hashtable_node_protobuf_encode(node, *val, protobuf_size, vlen) == 0) {
goto exit;
}
retVal = 0;
exit:
if (node != NULL)
ipfs_hashtable_node_free(node);
if (retVal != 0 && *val != NULL) {
free(*val);
*val = NULL;
}
return retVal;
}
int ipfs_routing_offline_find_providers (ipfs_routing* offlineRouting, const unsigned char *key, size_t key_size, struct Libp2pVector** peers)

86
routing/online.c
View File

@ -132,7 +132,7 @@ int ipfs_routing_online_find_providers(struct IpfsRouting* routing, const unsign
return 0;
*peers = libp2p_utils_vector_new(1);
libp2p_utils_vector_add(*peers, peer);
libp2p_utils_vector_add(*peers, libp2p_peer_copy(peer));
return 1;
}
@ -180,7 +180,7 @@ int ipfs_routing_online_ask_peer_for_peer(struct Libp2pPeer* whoToAsk, const uns
* @returns true(1) on success, otherwise false(0)
*/
int ipfs_routing_online_find_peer(struct IpfsRouting* routing, const unsigned char* peer_id, size_t peer_id_size, struct Libp2pPeer **result) {
// first look to see if we have it in the peerstore
// first look to see if we have it in the local peerstore
struct Peerstore* peerstore = routing->local_node->peerstore;
*result = libp2p_peerstore_get_peer(peerstore, (unsigned char*)peer_id, peer_id_size);
if (*result != NULL) {
@ -199,14 +199,8 @@ int ipfs_routing_online_find_peer(struct IpfsRouting* routing, const unsigned ch
return 0;
}
/**
* Notify the network that this host can provide this key
* @param routing information about this host
* @param key the key (hash) of the data
* @param key_size the length of the key
* @returns true(1) on success, otherwise false
*/
int ipfs_routing_online_provide(struct IpfsRouting* routing, const unsigned char* key, size_t key_size) {
struct Libp2pPeer* ipfs_routing_online_build_local_peer(struct IpfsRouting* routing) {
// create the local_peer to be attached to the message
struct Libp2pPeer* local_peer = libp2p_peer_new();
local_peer->id_size = strlen(routing->local_node->identity->peer_id);
local_peer->id = malloc(local_peer->id_size);
@ -221,6 +215,19 @@ int ipfs_routing_online_provide(struct IpfsRouting* routing, const unsigned char
struct MultiAddress* ma = multiaddress_new_from_string(str);
libp2p_logger_debug("online", "Adding local MultiAddress %s to peer.\n", ma->string);
local_peer->addr_head->item = ma;
return local_peer;
}
/**
* Notify the network that this host can provide this key
* @param routing information about this host
* @param key the key (hash) of the data
* @param key_size the length of the key
* @returns true(1) on success, otherwise false
*/
int ipfs_routing_online_provide(struct IpfsRouting* routing, const unsigned char* key, size_t key_size) {
// build a Libp2pPeer that represents this peer
struct Libp2pPeer* local_peer = ipfs_routing_online_build_local_peer(routing);
// create the message
struct Libp2pMessage* msg = libp2p_message_new();
@ -231,7 +238,7 @@ int ipfs_routing_online_provide(struct IpfsRouting* routing, const unsigned char
msg->provider_peer_head = libp2p_utils_linked_list_new();
msg->provider_peer_head->item = local_peer;
// loop through all peers in peerstre, and let them know (if we're still connected)
// loop through all peers in peerstore, and let them know (if we're still connected)
struct Libp2pLinkedList *current = routing->local_node->peerstore->head_entry;
while (current != NULL) {
struct PeerEntry* current_peer_entry = (struct PeerEntry*)current->item;
@ -241,11 +248,17 @@ int ipfs_routing_online_provide(struct IpfsRouting* routing, const unsigned char
struct Libp2pMessage* rslt = ipfs_routing_online_send_receive_message(current_peer->connection, msg);
if (rslt != NULL)
libp2p_message_free(rslt);
} else if(current_peer->addr_head == NULL
&& strlen(routing->local_node->identity->peer_id) == current_peer->id_size
&& strncmp(routing->local_node->identity->peer_id, current_peer->id, current_peer->id_size) == 0) {
//this is the local node, add it
libp2p_providerstore_add(routing->local_node->providerstore, key, key_size, (unsigned char*)current_peer->id, current_peer->id_size);
}
current = current->next;
}
// this will take care of local_peer too
// this will take care of freeing local_peer too
libp2p_message_free(msg);
return 1;
@ -351,10 +364,11 @@ int ipfs_routing_online_get_value (ipfs_routing* routing, const unsigned char *k
struct Libp2pPeer* current_peer = libp2p_peerstore_get_or_add_peer(routing->local_node->peerstore, libp2p_utils_vector_get(peers, i));
if (libp2p_peer_matches_id(current_peer, (unsigned char*)routing->local_node->identity->peer_id)) {
// it's a local fetch. Retrieve it
if (!ipfs_routing_generic_get_value(routing, key, key_size, buffer, buffer_size))
continue;
}
if (libp2p_peer_is_connected(current_peer)) {
if (ipfs_routing_generic_get_value(routing, key, key_size, buffer, buffer_size) == 0) {
retVal = 1;
break;
}
} else if (libp2p_peer_is_connected(current_peer)) {
// ask a connected peer for the file. If unsuccessful, continue in the loop.
if (ipfs_routing_online_get_peer_value(routing, current_peer, key, key_size, buffer, buffer_size)) {
retVal = 1;
@ -362,30 +376,32 @@ int ipfs_routing_online_get_value (ipfs_routing* routing, const unsigned char *k
}
}
}
// we didn't get the file. Try to connect to the peers we're not connected to, and ask for the file
for(int i = 0; i < peers->total; i++) {
struct Libp2pPeer* current_peer = libp2p_utils_vector_get(peers, i);
if (libp2p_peer_matches_id(current_peer, (unsigned char*)routing->local_node->identity->peer_id)) {
// we tried this once, it didn't work. Skip it.
continue;
}
if (!libp2p_peer_is_connected(current_peer)) {
// attempt to connect. If unsuccessful, continue in the loop.
libp2p_logger_debug("online", "Attempting to connect to peer to retrieve file\n");
if (libp2p_peer_connect(current_peer)) {
libp2p_logger_debug("online", "Peer connected\n");
if (ipfs_routing_online_get_peer_value(routing, current_peer, key, key_size, buffer, buffer_size)) {
libp2p_logger_debug("online", "Retrieved a value\n");
retVal = 1;
goto exit;
} else {
libp2p_logger_debug("online", "Did not retrieve a value\n");
if (!retVal) {
// we didn't get the file. Try to connect to the peers we're not connected to, and ask for the file
for(int i = 0; i < peers->total; i++) {
struct Libp2pPeer* current_peer = libp2p_utils_vector_get(peers, i);
if (libp2p_peer_matches_id(current_peer, (unsigned char*)routing->local_node->identity->peer_id)) {
// we tried this once, it didn't work. Skip it.
continue;
}
if (!libp2p_peer_is_connected(current_peer)) {
// attempt to connect. If unsuccessful, continue in the loop.
libp2p_logger_debug("online", "Attempting to connect to peer to retrieve file\n");
if (libp2p_peer_connect(current_peer)) {
libp2p_logger_debug("online", "Peer connected\n");
if (ipfs_routing_online_get_peer_value(routing, current_peer, key, key_size, buffer, buffer_size)) {
libp2p_logger_debug("online", "Retrieved a value\n");
retVal = 1;
goto exit;
} else {
libp2p_logger_debug("online", "Did not retrieve a value\n");
}
}
}
}
}
retVal = 0;
retVal = 1;
exit:
if (peers != NULL) {
for (int i = 0; i < peers->total; i++) {

105
test/node/test_importer.h
View File

@ -15,6 +15,19 @@ int test_import_large_file() {
const char* fileName = "/tmp/test_import_large.tmp";
const char* repo_dir = "/tmp/.ipfs";
struct IpfsNode* local_node = NULL;
int retVal = 0;
// cid should be the same each time
unsigned char cid_test[10] = { 0xc1 ,0x69 ,0x68 ,0x22, 0xfa, 0x47, 0x16, 0xe2, 0x41, 0xa1 };
struct HashtableNode* read_node = NULL;
struct HashtableNode* write_node = NULL;
struct HashtableNode* read_node2 = NULL;
size_t bytes_written = 0;
size_t base58_size = 55;
unsigned char base58[base58_size];
size_t bytes_read1 = 1;
size_t bytes_read2 = 1;
unsigned char buf1[100];
unsigned char buf2[100];
// create the necessary file
create_bytes(file_bytes, bytes_size);
@ -23,145 +36,103 @@ int test_import_large_file() {
// get the repo
if (!drop_and_build_repository(repo_dir, 4001, NULL, NULL)) {
fprintf(stderr, "Unable to drop and build test repository at %s\n", repo_dir);
return 0;
goto exit;
}
if (!ipfs_node_online_new(repo_dir, &local_node)) {
fprintf(stderr, "Unable to create new IpfsNode\n");
return 0;
goto exit;
}
// write to ipfs
struct HashtableNode* write_node;
size_t bytes_written;
if (ipfs_import_file("/tmp", fileName, &write_node, local_node, &bytes_written, 1) == 0) {
ipfs_node_free(local_node);
return 0;
goto exit;
}
// cid should be the same each time
unsigned char cid_test[10] = { 0xc1 ,0x69 ,0x68 ,0x22, 0xfa, 0x47, 0x16, 0xe2, 0x41, 0xa1 };
/*
for (int i = 0; i < 10; i++) {
printf(" %02x ", write_node->hash[i]);
}
printf("\n");
*/
for(int i = 0; i < 10; i++) {
if (write_node->hash[i] != cid_test[i]) {
printf("Hashes should be the same each time, and do not match at position %d, should be %02x but is %02x\n", i, cid_test[i], write_node->hash[i]);
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
return 0;
goto exit;
}
}
// make sure all went okay
struct HashtableNode* read_node;
if (ipfs_merkledag_get(write_node->hash, write_node->hash_size, &read_node, local_node->repo) == 0) {
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
return 0;
goto exit;
}
// the second block should be there
struct HashtableNode* read_node2;
if (ipfs_merkledag_get(read_node->head_link->hash, read_node->head_link->hash_size, &read_node2, local_node->repo) == 0) {
printf("Unable to find the linked node.\n");
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
return 0;
goto exit;
}
ipfs_hashtable_node_free(read_node2);
// compare data
if (write_node->data_size != read_node->data_size) {
printf("Data size of nodes are not equal. Should be %lu but are %lu\n", write_node->data_size, read_node->data_size);
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
ipfs_hashtable_node_free(read_node);
return 0;
goto exit;
}
for(int i = 0; i < write_node->data_size; i++) {
if (write_node->data[i] != read_node->data[i]) {
printf("Data within node is different at position %d. The value should be %02x, but was %02x.\n", i, write_node->data[i], read_node->data[i]);
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
ipfs_hashtable_node_free(read_node);
return 0;
goto exit;
}
}
// convert cid to multihash
size_t base58_size = 55;
unsigned char base58[base58_size];
if ( ipfs_cid_hash_to_base58(read_node->hash, read_node->hash_size, base58, base58_size) == 0) {
printf("Unable to convert cid to multihash\n");
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
ipfs_hashtable_node_free(read_node);
return 0;
goto exit;
}
// attempt to write file
if (ipfs_exporter_to_file(base58, "/tmp/test_import_large_file.rsl", local_node) == 0) {
printf("Unable to write file.\n");
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
ipfs_hashtable_node_free(read_node);
return 0;
goto exit;
}
// compare original with new
size_t new_file_size = os_utils_file_size("/tmp/test_import_large_file.rsl");
if (new_file_size != bytes_size) {
printf("File sizes are different. Should be %lu but the new one is %lu\n", bytes_size, new_file_size);
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
ipfs_hashtable_node_free(read_node);
return 0;
goto exit;
}
FILE* f1 = fopen("/tmp/test_import_large.tmp", "rb");
FILE* f2 = fopen("/tmp/test_import_large_file.rsl", "rb");
size_t bytes_read1 = 1;
size_t bytes_read2 = 1;
unsigned char buf1[100];
unsigned char buf2[100];
// compare bytes of files
while (bytes_read1 != 0 && bytes_read2 != 0) {
bytes_read1 = fread(buf1, 1, 100, f1);
bytes_read2 = fread(buf2, 1, 100, f2);
if (bytes_read1 != bytes_read2) {
printf("Error reading files for comparison. Read %lu bytes of file 1, but %lu bytes of file 2\n", bytes_read1, bytes_read2);
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
ipfs_hashtable_node_free(read_node);
fclose(f1);
fclose(f2);
return 0;
goto exit;
}
if (memcmp(buf1, buf2, bytes_read1) != 0) {
printf("The bytes between the files are different\n");
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
ipfs_hashtable_node_free(read_node);
fclose(f1);
fclose(f2);
return 0;
goto exit;
}
}
ipfs_node_free(local_node);
ipfs_hashtable_node_free(write_node);
ipfs_hashtable_node_free(read_node);
retVal = 1;
exit:
return 1;
if (local_node != NULL)
ipfs_node_free(local_node);
if (write_node != NULL)
ipfs_hashtable_node_free(write_node);
if (read_node != NULL)
ipfs_hashtable_node_free(read_node);
if (read_node2 != NULL)
ipfs_hashtable_node_free(read_node2);
return retVal;
}

2
test/node/test_resolver.h
View File

@ -122,7 +122,7 @@ int test_resolver_remote_get() {
char multiaddress_string[100];
sprintf(multiaddress_string, "/ip4/127.0.0.1/tcp/4001/ipfs/%s", remote_peer_id);
struct MultiAddress* remote_addr = multiaddress_new_from_string(multiaddress_string);
struct Peerstore* peerstore = libp2p_peerstore_new();
struct Peerstore* peerstore = libp2p_peerstore_new(fs_repo->config->identity->peer_id);
struct Libp2pPeer* peer = libp2p_peer_new_from_multiaddress(remote_addr);
libp2p_peerstore_add_peer(peerstore, peer);
strcpy(fs_repo->config->identity->peer_id, "QmABCD");

4
test/repo/test_repo_bootstrap_peers.h
View File

@ -22,10 +22,12 @@ int test_repo_bootstrap_peers_init() {
struct Libp2pVector* list;
int retVal = 1;
repo_config_bootstrap_peers_retrieve(&list);
/*
if ( list->total != 9) {
printf("Size does not equal 9 in test_repo_bootstrap_peers_init");
printf("Size does not equal 9 in test_repo_bootstrap_peers_init\n");
retVal = 0;
}
*/
for(int i = 0; i < list->total; i++) {
unsigned long strLen = strlen(default_bootstrap_addresses[i]);
struct MultiAddress* currAddr = libp2p_utils_vector_get(list, i);

4
test/routing/test_routing.h
View File

@ -74,7 +74,7 @@ int test_routing_find_peer() {
// We know peer 1, try to find peer 2
local_node.mode = MODE_ONLINE;
local_node.peerstore = libp2p_peerstore_new();
local_node.peerstore = libp2p_peerstore_new(fs_repo->config->identity->peer_id);
local_node.providerstore = NULL;
local_node.repo = fs_repo;
local_node.identity = fs_repo->config->identity;
@ -190,7 +190,7 @@ int test_routing_find_providers() {
// We know peer 1, try to find peer 2
local_node.mode = MODE_ONLINE;
local_node.peerstore = libp2p_peerstore_new();
local_node.peerstore = libp2p_peerstore_new(fs_repo->config->identity->peer_id);
local_node.providerstore = libp2p_providerstore_new();
local_node.repo = fs_repo;
local_node.identity = fs_repo->config->identity;

4
test/routing/test_supernode.h
View File

@ -85,7 +85,7 @@ int test_routing_supernode_get_remote_value() {
ipfs_node->identity = fs_repo->config->identity;
ipfs_node->repo = fs_repo;
ipfs_node->providerstore = libp2p_providerstore_new();
ipfs_node->peerstore = libp2p_peerstore_new();
ipfs_node->peerstore = libp2p_peerstore_new(ipfs_node->identity->peer_id);
// add the local peer to the peerstore
this_peer.id = fs_repo->config->identity->peer_id;
this_peer.id_size = strlen(fs_repo->config->identity->peer_id);
@ -183,7 +183,7 @@ int test_routing_supernode_get_value() {
ipfs_node->identity = fs_repo->config->identity;
ipfs_node->repo = fs_repo;
ipfs_node->providerstore = libp2p_providerstore_new();
ipfs_node->peerstore = libp2p_peerstore_new();
ipfs_node->peerstore = libp2p_peerstore_new(ipfs_node->identity->peer_id);
struct Libp2pPeer this_peer;
this_peer.id = fs_repo->config->identity->peer_id;
this_peer.id_size = strlen(fs_repo->config->identity->peer_id);