From 22a782d9f195f7b38886f4739fe4accf49007cd5 Mon Sep 17 00:00:00 2001 From: John Jones Date: Thu, 20 Apr 2017 17:55:18 -0500 Subject: [PATCH] Major changes to retrieving files Working on transfer of big files --- Makefile | 2 +- db/Makefile | 2 +- db/filestore.c | 41 ++++++++++++++++++ include/libp2p/conn/session.h | 3 ++ include/libp2p/db/filestore.h | 50 ++++++++++++++++++++++ include/libp2p/peer/peer.h | 15 +++++++ include/libp2p/peer/peerstore.h | 9 ++++ include/libp2p/peer/providerstore.h | 2 +- peer/peer.c | 34 +++++++++++++++ peer/peerstore.c | 66 +++++++++++++++++++++-------- peer/providerstore.c | 2 +- routing/dht_protocol.c | 32 +++++++++----- utils/logger.c | 17 +++++--- 13 files changed, 237 insertions(+), 38 deletions(-) create mode 100644 db/filestore.c create mode 100644 include/libp2p/db/filestore.h diff --git a/Makefile b/Makefile index 3bd3443..96aacf5 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ OBJS = \ conn/*.o \ crypto/*.o \ crypto/encoding/*.o \ - db/datastore.o \ + db/*.o \ thirdparty/mbedtls/*.o \ hashmap/hashmap.o \ net/*.o \ diff --git a/db/Makefile b/db/Makefile index 58c96d0..ab7f456 100644 --- a/db/Makefile +++ b/db/Makefile @@ -2,7 +2,7 @@ CC = gcc CFLAGS = -O0 -I../include -I../../c-protobuf -I../../c-multihash/include -I../../c-multiaddr/include -g3 -std=c99 LFLAGS = DEPS = -OBJS = datastore.o +OBJS = datastore.o filestore.o %.o: %.c $(DEPS) $(CC) -c -o $@ $< $(CFLAGS) diff --git a/db/filestore.c b/db/filestore.c new file mode 100644 index 0000000..89f474b --- /dev/null +++ b/db/filestore.c @@ -0,0 +1,41 @@ +#include +#include + +#include "libp2p/db/filestore.h" +#include "libp2p/os/utils.h" + +/*** + * initialize the structure of the filestore + * @param filestore the struct to initialize + * @returns true(1) on success + */ +int libp2p_filestore_init(struct Filestore* datastore, const char* config_root) { + return 1; +} + +/*** + * initialize the structure of the filestore + * @param filestore the struct to initialize + * @returns true(1) on success + */ +struct Filestore* libp2p_filestore_new() { + struct Filestore* f = malloc(sizeof(struct Filestore)); + if (f == NULL) + return 0; + f->handle = NULL; + f->node_get = NULL; + return f; +} + +/*** + * deallocate the memory and clear resources from a filestore_init + * @param filestore the struct to deallocate + * @returns true(1) + */ +int libp2p_filestore_free(struct Filestore* filestore) { + if (filestore != NULL) + { + free(filestore); + } + return 1; +} diff --git a/include/libp2p/conn/session.h b/include/libp2p/conn/session.h index 6bda610..69859f8 100644 --- a/include/libp2p/conn/session.h +++ b/include/libp2p/conn/session.h @@ -2,6 +2,8 @@ #include "libp2p/crypto/key.h" #include "libp2p/db/datastore.h" +#include "libp2p/db/filestore.h" + /*** * Holds the details of communication between two hosts */ @@ -18,6 +20,7 @@ struct SessionContext { struct Stream* secure_stream; struct Stream* default_stream; struct Datastore* datastore; + struct Filestore* filestore; // filled in during negotiations char* chosen_curve; char* chosen_cipher; diff --git a/include/libp2p/db/filestore.h b/include/libp2p/db/filestore.h new file mode 100644 index 0000000..76230f5 --- /dev/null +++ b/include/libp2p/db/filestore.h @@ -0,0 +1,50 @@ +#pragma once + +#include + +/*** + * Interface to data storage + */ + + +struct Filestore { + + // generic connection and status variables for the datastore + void* handle; // a handle to the filesstore (a FSRepo struct) + + // function pointers for datastore operations + /** + * Retrieves a protobuf'd node from the disk + * @param key the key + * @param key_size the key size + * @param data the protobuf'd results + * @param data_size the size of the results + * @param filestore a reference to the filestore struct + */ + int (*node_get)(const unsigned char* key, size_t key_size, + void** data, size_t *data_size, const struct Filestore* filestore); +}; + +/*** + * Initialize the structure of the filestore to default settings. Used for + * creating a new filestore on the disk. + * @param filestore the struct to initialize + * @param config_root the path to the root of IPFS + * @returns true(1) on success + */ +int libp2p_filestore_init(struct Filestore* filestore, const char* config_root); + +/*** + * initialize the structure of the filestore + * @param filestore the struct to initialize + * @returns true(1) on success + */ +struct Filestore* libp2p_filestore_new(); + + +/*** + * deallocate the memory and clear resources from a filestore_init + * @param filestore the struct to deallocate + * @returns true(1) + */ +int libp2p_filestore_free(struct Filestore* datastore); diff --git a/include/libp2p/peer/peer.h b/include/libp2p/peer/peer.h index 4f96960..a17aeb4 100644 --- a/include/libp2p/peer/peer.h +++ b/include/libp2p/peer/peer.h @@ -56,6 +56,21 @@ int libp2p_peer_connect(struct Libp2pPeer* peer); */ struct Libp2pPeer* libp2p_peer_copy(struct Libp2pPeer* in); +/*** + * Determine if the passed in peer and id match + * @param in the peer to check + * @param peer_id peer id, zero terminated string + * @returns true if peer matches + */ +int libp2p_peer_matches_id(struct Libp2pPeer* in, const unsigned char* peer_id); + +/*** + * Determine if we are currently connected to this peer + * @param in the peer to check + * @returns true(1) if connected + */ +int libp2p_peer_is_connected(struct Libp2pPeer* in); + /** * Get an estimate of the necessary size of the buffer to protobuf a particular peer * @param in the peer to examine diff --git a/include/libp2p/peer/peerstore.h b/include/libp2p/peer/peerstore.h index b34aa71..6073486 100644 --- a/include/libp2p/peer/peerstore.h +++ b/include/libp2p/peer/peerstore.h @@ -77,3 +77,12 @@ struct PeerEntry* libp2p_peerstore_get_peer_entry(struct Peerstore* peerstore, c * @returns the Libp2pPeer struct if found, otherwise NULL */ struct Libp2pPeer* libp2p_peerstore_get_peer(struct Peerstore* peerstore, const unsigned char* peer_id, size_t peer_id_size); + +/** + * Look for this peer in the peerstore. If it is found, return a reference to that object. + * If it is not found, add it, and return a reference to the new copy + * @param peerstore the peerstore to search + * @param in the peer to search for + */ +struct Libp2pPeer* libp2p_peerstore_get_or_add_peer(struct Peerstore* peerstore, struct Libp2pPeer* in); + diff --git a/include/libp2p/peer/providerstore.h b/include/libp2p/peer/providerstore.h index ebc216e..9ded207 100644 --- a/include/libp2p/peer/providerstore.h +++ b/include/libp2p/peer/providerstore.h @@ -34,4 +34,4 @@ void libp2p_providerstore_free(struct ProviderStore* in); int libp2p_providerstore_add(struct ProviderStore* store, unsigned char* hash, int hash_size, unsigned char* peer_id, int peer_id_size); -int libp2p_providerstore_get(struct ProviderStore* store, unsigned char* hash, int hash_size, unsigned char** peer_id, int *peer_id_size); +int libp2p_providerstore_get(struct ProviderStore* store, const unsigned char* hash, int hash_size, unsigned char** peer_id, int *peer_id_size); diff --git a/peer/peer.c b/peer/peer.c index 72a179e..8c2784f 100644 --- a/peer/peer.c +++ b/peer/peer.c @@ -5,6 +5,7 @@ #include "multiaddr/multiaddr.h" #include "protobuf.h" #include "libp2p/net/multistream.h" +#include "libp2p/utils/logger.h" /** * create a new Peer struct @@ -99,6 +100,11 @@ struct Libp2pPeer* libp2p_peer_new_from_data(const char* id, size_t id_size, con void libp2p_peer_free(struct Libp2pPeer* in) { if (in != NULL) { + if (in->addr_head != NULL && in->addr_head->item != NULL) { + libp2p_logger_debug("peer", "Freeing peer %s\n", ((struct MultiAddress*)in->addr_head->item)->string); + } else { + libp2p_logger_debug("peer", "Freeing peer with no multiaddress.\n"); + } if (in->id != NULL) free(in->id); if (in->connection != NULL) { @@ -153,6 +159,34 @@ struct Libp2pPeer* libp2p_peer_copy(struct Libp2pPeer* in) { return out; } +/*** + * Determine if the passed in peer and id match + * @param in the peer to check + * @param peer_id peer id, zero terminated string + * @returns true if peer matches + */ +int libp2p_peer_matches_id(struct Libp2pPeer* in, const unsigned char* peer_id) { + if (strlen(peer_id) == in->id_size) { + if (strncmp(in->id, peer_id, in->id_size) == 0) + return 1; + } + return 0; +} + +/*** + * Determine if we are currently connected to this peer + * @param in the peer to check + * @returns true(1) if connected + */ +int libp2p_peer_is_connected(struct Libp2pPeer* in) { + if (in->connection_type == CONNECTION_TYPE_CONNECTED) { + if (in->connection == NULL) { + in->connection_type = CONNECTION_TYPE_NOT_CONNECTED; + } + } + return in->connection_type == CONNECTION_TYPE_CONNECTED; +} + size_t libp2p_peer_protobuf_encode_size(struct Libp2pPeer* in) { int sz = 0; if (in != NULL) { diff --git a/peer/peerstore.c b/peer/peerstore.c index a288e2d..6af0b1e 100644 --- a/peer/peerstore.c +++ b/peer/peerstore.c @@ -39,6 +39,7 @@ struct Peerstore* libp2p_peerstore_new() { struct Peerstore* out = (struct Peerstore*)malloc(sizeof(struct Peerstore)); if (out != NULL) { out->head_entry = NULL; + out->last_entry = NULL; } return out; } @@ -56,9 +57,9 @@ int libp2p_peerstore_free(struct Peerstore* in) { next = current->next; libp2p_peer_entry_free((struct PeerEntry*)current->item); current->item = NULL; - libp2p_utils_linked_list_free(current); current = next; } + libp2p_utils_linked_list_free(in->head_entry); free(in); } return 1; @@ -96,24 +97,37 @@ int libp2p_peerstore_add_peer_entry(struct Peerstore* peerstore, struct PeerEntr * @returns true(1) on success, otherwise false */ int libp2p_peerstore_add_peer(struct Peerstore* peerstore, struct Libp2pPeer* peer) { - // first check to see if it exists. If it does, return TRUE - if (libp2p_peerstore_get_peer_entry(peerstore, peer->id, peer->id_size) != NULL) - return 1; + int retVal = 0; - char peer_id[peer->id_size + 1]; - memcpy(peer_id, peer->id, peer->id_size); - peer_id[peer->id_size] = 0; - char* address = ((struct MultiAddress*)peer->addr_head->item)->string; - libp2p_logger_debug("peerstore", "Adding peer %s with address %s to peer store\n", peer_id, address); - struct PeerEntry* peer_entry = libp2p_peer_entry_new(); - if (peer_entry == NULL) { - return 0; + char* ma_string = ""; + if (peer != NULL && peer->addr_head != NULL && peer->addr_head->item != NULL) { + ma_string = ((struct MultiAddress*)peer->addr_head->item)->string; + } + // first check to see if it exists. If it does, return TRUE + if (libp2p_peerstore_get_peer_entry(peerstore, peer->id, peer->id_size) != NULL) { + libp2p_logger_debug("peerstore", "Attempted to add %s to peerstore, but already there.\n", ma_string); + return 1; + } + + if (peer->id_size > 0) { + char peer_id[peer->id_size + 1]; + memcpy(peer_id, peer->id, peer->id_size); + peer_id[peer->id_size] = 0; + if (peer->addr_head != NULL) { + char* address = ((struct MultiAddress*)peer->addr_head->item)->string; + libp2p_logger_debug("peerstore", "Adding peer %s with address %s to peer store\n", peer_id, address); + } + struct PeerEntry* peer_entry = libp2p_peer_entry_new(); + if (peer_entry == NULL) { + return 0; + } + peer_entry->peer = libp2p_peer_copy(peer); + if (peer_entry->peer == NULL) + return 0; + retVal = libp2p_peerstore_add_peer_entry(peerstore, peer_entry); + libp2p_peer_entry_free(peer_entry); + libp2p_logger_debug("peerstore", "Adding peer %s to peerstore was a success\n", peer_id); } - peer_entry->peer = libp2p_peer_copy(peer); - if (peer_entry->peer == NULL) - return 0; - int retVal = libp2p_peerstore_add_peer_entry(peerstore, peer_entry); - libp2p_peer_entry_free(peer_entry); return retVal; } @@ -151,3 +165,21 @@ struct Libp2pPeer* libp2p_peerstore_get_peer(struct Peerstore* peerstore, const return NULL; return entry->peer; } + +/** + * Look for this peer in the peerstore. If it is found, return a reference to that object. + * If it is not found, add it, and return a reference to the new copy + * @param peerstore the peerstore to search + * @param in the peer to search for + */ +struct Libp2pPeer* libp2p_peerstore_get_or_add_peer(struct Peerstore* peerstore, struct Libp2pPeer* in) { + struct Libp2pPeer* out = libp2p_peerstore_get_peer(peerstore, (unsigned char*)in->id, in->id_size); + if (out != NULL) + return out; + + // we didn't find it. attempt to add + if (!libp2p_peerstore_add_peer(peerstore, in)) + return NULL; + + return libp2p_peerstore_get_peer(peerstore, (unsigned char*)in->id, in->id_size); +} diff --git a/peer/providerstore.c b/peer/providerstore.c index c58d74a..15b9a44 100644 --- a/peer/providerstore.c +++ b/peer/providerstore.c @@ -80,7 +80,7 @@ int libp2p_providerstore_add(struct ProviderStore* store, unsigned char* hash, i return 1; } -int libp2p_providerstore_get(struct ProviderStore* store, unsigned char* hash, int hash_size, unsigned char** peer_id, int *peer_id_size) { +int libp2p_providerstore_get(struct ProviderStore* store, const unsigned char* hash, int hash_size, unsigned char** peer_id, int *peer_id_size) { struct ProviderEntry* current = NULL; for (int i = 0; i < store->provider_entries->total; i++) { current = (struct ProviderEntry*)libp2p_utils_vector_get(store->provider_entries, i); diff --git a/routing/dht_protocol.c b/routing/dht_protocol.c index 1ca8ba2..b5d90c3 100644 --- a/routing/dht_protocol.c +++ b/routing/dht_protocol.c @@ -5,6 +5,7 @@ #include "libp2p/routing/dht_protocol.h" #include "libp2p/record/message.h" #include "libp2p/utils/logger.h" +#include "libp2p/conn/session.h" /*** @@ -97,12 +98,15 @@ int libp2p_routing_dht_handle_get_providers(struct SessionContext* session, stru // Can I provide it? if (libp2p_providerstore_get(providerstore, message->key, message->key_size, &peer_id, &peer_id_size)) { + libp2p_logger_debug("dht_protocol", "I can provide a provider for this key.\n"); // we have a peer id, convert it to a peer object struct Libp2pPeer* peer = libp2p_peerstore_get_peer(peerstore, peer_id, peer_id_size); if (peer != NULL) { message->provider_peer_head = libp2p_utils_linked_list_new(); - message->provider_peer_head->item = peer; + message->provider_peer_head->item = libp2p_peer_copy(peer); } + } else { + libp2p_logger_debug("dht_protocol", "I cannot provide a provider for this key.\n"); } free(peer_id); // TODO: find closer peers @@ -113,8 +117,10 @@ int libp2p_routing_dht_handle_get_providers(struct SessionContext* session, stru } */ if (message->provider_peer_head != NULL) { + libp2p_logger_debug("dht_protocol", "GetProviders: We have a peer. Sending it back\n"); // protobuf it and send it back if (!libp2p_routing_dht_protobuf_message(message, results, results_size)) { + libp2p_logger_error("dht_protocol", "GetProviders: Error protobufing results\n"); return 0; } } @@ -162,6 +168,7 @@ int libp2p_routing_dht_handle_add_provider(struct SessionContext* session, struc if (message->record != NULL && message->record->author != NULL && message->record->author_size > 0 && message->key != NULL && message->key_size > 0) */ + struct Libp2pLinkedList* current = message->provider_peer_head; if (current == NULL) { libp2p_logger_error("dht_protocol", "Provider has no peer.\n"); @@ -199,7 +206,7 @@ int libp2p_routing_dht_handle_add_provider(struct SessionContext* session, struc new_head->next = peer->addr_head; peer->addr_head = new_head; // now add the peer to the peerstore - libp2p_logger_debug("dht_protocol", "About to add peer to peerstore\n"); + libp2p_logger_debug("dht_protocol", "About to add peer %s to peerstore\n", peer_ma->string); if (!libp2p_peerstore_add_peer(peerstore, peer)) goto exit; libp2p_logger_debug("dht_protocol", "About to add key to providerstore\n"); @@ -240,14 +247,19 @@ int libp2p_routing_dht_handle_add_provider(struct SessionContext* session, struc */ int libp2p_routing_dht_handle_get_value(struct SessionContext* session, struct Libp2pMessage* message, struct Peerstore* peerstore, struct ProviderStore* providerstore, unsigned char** result_buffer, size_t *result_buffer_size) { - //TODO: implement this + struct Datastore* datastore = session->datastore; - size_t data_size = 65535; - char* data = malloc(data_size); - if (!datastore->datastore_get(message->key, message->key_size, data, data_size, &data_size, datastore)) { - free(data); - return 0; + struct Filestore* filestore = session->filestore; + size_t data_size = 0; + unsigned char* data = NULL; + + // We need to get the data from the disk + if(!filestore->node_get(message->key, message->key_size, (void**)&data, &data_size, filestore)) { + libp2p_logger_debug("dht_protocol", "handle_get_value: Unable to get key from filestore\n"); } + + libp2p_logger_debug("dht_protocol", "handle_get_value: value retrieved from the datastore\n"); + struct Libp2pRecord *record = libp2p_record_new(); record->key_size = message->key_size; record->key = malloc(record->key_size); @@ -352,11 +364,11 @@ int libp2p_routing_dht_handle_message(struct SessionContext* session, struct Pee } // if we have something to send, send it. if (result_buffer != NULL) { - libp2p_logger_debug("dht_protocol", "Sending message back to caller\n"); + libp2p_logger_debug("dht_protocol", "Sending message back to caller. Message type: %d\n", message->message_type); if (!session->default_stream->write(session, result_buffer, result_buffer_size)) goto exit; } else { - libp2p_logger_debug("dht_protocol", "Nothing to send back. Kademlia call has been handled\n"); + libp2p_logger_debug("dht_protocol", "DhtHandleMessage: Nothing to send back. Kademlia call has been handled. Message type: %d\n", message->message_type); } retVal = 1; exit: diff --git a/utils/logger.c b/utils/logger.c index fb41592..36e490e 100644 --- a/utils/logger.c +++ b/utils/logger.c @@ -17,9 +17,6 @@ struct Libp2pVector* logger_classes = NULL; */ void libp2p_logger_init() { logger_classes = libp2p_utils_vector_new(1); - libp2p_logger_add_class("secio"); - libp2p_logger_add_class("null"); - libp2p_logger_add_class("dht_protocol"); } /*** @@ -88,12 +85,18 @@ void libp2p_logger_log(const char* area, int log_level, const char* format, ...) void libp2p_logger_vlog(const char* area, int log_level, const char* format, va_list argptr) { if (!libp2p_logger_initialized()) libp2p_logger_init(); + // only allow a message if the message log level is less than the current loglevel if (log_level <= CURRENT_LOGLEVEL) { int found = 0; - for (int i = 0; i < logger_classes->total; i++) { - if (strcmp(libp2p_utils_vector_get(logger_classes, i), area) == 0) { - found = 1; - break; + // error should always be printed for now. We need to think about this more... + if (log_level <= LOGLEVEL_ERROR ) + found = 1; + else { + for (int i = 0; i < logger_classes->total; i++) { + if (strcmp(libp2p_utils_vector_get(logger_classes, i), area) == 0) { + found = 1; + break; + } } } if (found) {