Major changes to retrieving files

Working on transfer of big files

Makefile

@@ -6,7 +6,7 @@ OBJS = \
 	conn/*.o \
 	crypto/*.o \
 	crypto/encoding/*.o \
-	db/datastore.o \
+	db/*.o \
 	thirdparty/mbedtls/*.o \
 	hashmap/hashmap.o \
 	net/*.o \

db/Makefile

@@ -2,7 +2,7 @@ CC = gcc
 CFLAGS = -O0 -I../include -I../../c-protobuf -I../../c-multihash/include -I../../c-multiaddr/include -g3 -std=c99
 LFLAGS =
 DEPS = 
-OBJS = datastore.o
+OBJS = datastore.o filestore.o
 
 %.o: %.c $(DEPS)
 	$(CC) -c -o $@ $< $(CFLAGS)

db/filestore.c

@@ -0,0 +1,41 @@
+#include <stdlib.h>
+#include <string.h>
+
+#include "libp2p/db/filestore.h"
+#include "libp2p/os/utils.h"
+
+/***
+ * initialize the structure of the filestore
+ * @param filestore the struct to initialize
+ * @returns true(1) on success
+ */
+int libp2p_filestore_init(struct Filestore* datastore, const char* config_root) {
+	return 1;
+}
+
+/***
+ * initialize the structure of the filestore
+ * @param filestore the struct to initialize
+ * @returns true(1) on success
+ */
+struct Filestore* libp2p_filestore_new() {
+	struct Filestore* f = malloc(sizeof(struct Filestore));
+	if (f == NULL)
+		return 0;
+	f->handle = NULL;
+	f->node_get = NULL;
+	return f;
+}
+
+/***
+ * deallocate the memory and clear resources from a filestore_init
+ * @param filestore the struct to deallocate
+ * @returns true(1)
+ */
+int libp2p_filestore_free(struct Filestore* filestore) {
+	if (filestore != NULL)
+	{
+		free(filestore);
+	}
+	return 1;
+}

include/libp2p/conn/session.h

@@ -2,6 +2,8 @@
 
 #include "libp2p/crypto/key.h"
 #include "libp2p/db/datastore.h"
+#include "libp2p/db/filestore.h"
+
 /***
  * Holds the details of communication between two hosts
  */
@@ -18,6 +20,7 @@ struct SessionContext {
 	struct Stream* secure_stream;
 	struct Stream* default_stream;
 	struct Datastore* datastore;
+	struct Filestore* filestore;
 	// filled in during negotiations
 	char* chosen_curve;
 	char* chosen_cipher;

include/libp2p/db/filestore.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <stdint.h>
+
+/***
+ * Interface to data storage
+ */
+
+
+struct Filestore {
+
+	// generic connection and status variables for the datastore
+	void* handle; // a handle to the filesstore (a FSRepo struct)
+
+	// function pointers for datastore operations
+	/**
+	 * Retrieves a protobuf'd node from the disk
+	 * @param key the key
+	 * @param key_size the key size
+	 * @param data the protobuf'd results
+	 * @param data_size the size of the results
+	 * @param filestore a reference to the filestore struct
+	 */
+	int (*node_get)(const unsigned char* key, size_t key_size,
+			void** data, size_t *data_size, const struct Filestore* filestore);
+};
+
+/***
+ * Initialize the structure of the filestore to default settings. Used for
+ * creating a new filestore on the disk.
+ * @param filestore the struct to initialize
+ * @param config_root the path to the root of IPFS
+ * @returns true(1) on success
+ */
+int libp2p_filestore_init(struct Filestore* filestore, const char* config_root);
+
+/***
+ * initialize the structure of the filestore
+ * @param filestore the struct to initialize
+ * @returns true(1) on success
+ */
+struct Filestore* libp2p_filestore_new();
+
+
+/***
+ * deallocate the memory and clear resources from a filestore_init
+ * @param filestore the struct to deallocate
+ * @returns true(1)
+ */
+int libp2p_filestore_free(struct Filestore* datastore);

include/libp2p/peer/peer.h

@@ -56,6 +56,21 @@ int libp2p_peer_connect(struct Libp2pPeer* peer);
  */
 struct Libp2pPeer* libp2p_peer_copy(struct Libp2pPeer* in);
 
+/***
+ * Determine if the passed in peer and id match
+ * @param in the peer to check
+ * @param peer_id peer id, zero terminated string
+ * @returns true if peer matches
+ */
+int libp2p_peer_matches_id(struct Libp2pPeer* in, const unsigned char* peer_id);
+
+/***
+ * Determine if we are currently connected to this peer
+ * @param in the peer to check
+ * @returns true(1) if connected
+ */
+int libp2p_peer_is_connected(struct Libp2pPeer* in);
+
 /**
  * Get an estimate of the necessary size of the buffer to protobuf a particular peer
  * @param in the peer to examine

include/libp2p/peer/peerstore.h

@@ -77,3 +77,12 @@ struct PeerEntry* libp2p_peerstore_get_peer_entry(struct Peerstore* peerstore, c
  * @returns the Libp2pPeer struct if found, otherwise NULL
  */
 struct Libp2pPeer* libp2p_peerstore_get_peer(struct Peerstore* peerstore, const unsigned char* peer_id, size_t peer_id_size);
+
+/**
+ * Look for this peer in the peerstore. If it is found, return a reference to that object.
+ * If it is not found, add it, and return a reference to the new copy
+ * @param peerstore the peerstore to search
+ * @param in the peer to search for
+ */
+struct Libp2pPeer* libp2p_peerstore_get_or_add_peer(struct Peerstore* peerstore, struct Libp2pPeer* in);
+

include/libp2p/peer/providerstore.h

@@ -34,4 +34,4 @@ void libp2p_providerstore_free(struct ProviderStore* in);
 
 int libp2p_providerstore_add(struct ProviderStore* store, unsigned char* hash, int hash_size, unsigned char* peer_id, int peer_id_size);
 
-int libp2p_providerstore_get(struct ProviderStore* store, unsigned char* hash, int hash_size, unsigned char** peer_id, int *peer_id_size);
+int libp2p_providerstore_get(struct ProviderStore* store, const unsigned char* hash, int hash_size, unsigned char** peer_id, int *peer_id_size);

peer/peer.c

@@ -5,6 +5,7 @@
 #include "multiaddr/multiaddr.h"
 #include "protobuf.h"
 #include "libp2p/net/multistream.h"
+#include "libp2p/utils/logger.h"
 
 /**
  * create a new Peer struct
@@ -99,6 +100,11 @@ struct Libp2pPeer* libp2p_peer_new_from_data(const char* id, size_t id_size, con
 
 void libp2p_peer_free(struct Libp2pPeer* in) {
 	if (in != NULL) {
+		if (in->addr_head != NULL && in->addr_head->item != NULL) {
+			libp2p_logger_debug("peer", "Freeing peer %s\n", ((struct MultiAddress*)in->addr_head->item)->string);
+		} else {
+			libp2p_logger_debug("peer", "Freeing peer with no multiaddress.\n");
+		}
 		if (in->id != NULL)
 			free(in->id);
 		if (in->connection != NULL) {
@@ -153,6 +159,34 @@ struct Libp2pPeer* libp2p_peer_copy(struct Libp2pPeer* in) {
 	return out;
 }
 
+/***
+ * Determine if the passed in peer and id match
+ * @param in the peer to check
+ * @param peer_id peer id, zero terminated string
+ * @returns true if peer matches
+ */
+int libp2p_peer_matches_id(struct Libp2pPeer* in, const unsigned char* peer_id) {
+	if (strlen(peer_id) == in->id_size) {
+		if (strncmp(in->id, peer_id, in->id_size) == 0)
+			return 1;
+	}
+	return 0;
+}
+
+/***
+ * Determine if we are currently connected to this peer
+ * @param in the peer to check
+ * @returns true(1) if connected
+ */
+int libp2p_peer_is_connected(struct Libp2pPeer* in) {
+	if (in->connection_type == CONNECTION_TYPE_CONNECTED) {
+		if (in->connection == NULL) {
+			in->connection_type = CONNECTION_TYPE_NOT_CONNECTED;
+		}
+	}
+	return in->connection_type == CONNECTION_TYPE_CONNECTED;
+}
+
 size_t libp2p_peer_protobuf_encode_size(struct Libp2pPeer* in) {
 	int sz = 0;
 	if (in != NULL) {

peer/peerstore.c

@@ -39,6 +39,7 @@ struct Peerstore* libp2p_peerstore_new() {
 	struct Peerstore* out = (struct Peerstore*)malloc(sizeof(struct Peerstore));
 	if (out != NULL) {
 		out->head_entry = NULL;
+		out->last_entry = NULL;
 	}
 	return out;
 }
@@ -56,9 +57,9 @@ int libp2p_peerstore_free(struct Peerstore* in) {
 			next = current->next;
 			libp2p_peer_entry_free((struct PeerEntry*)current->item);
 			current->item = NULL;
-			libp2p_utils_linked_list_free(current);
 			current = next;
 		}
+		libp2p_utils_linked_list_free(in->head_entry);
 		free(in);
 	}
 	return 1;
@@ -96,15 +97,26 @@ int libp2p_peerstore_add_peer_entry(struct Peerstore* peerstore, struct PeerEntr
  * @returns true(1) on success, otherwise false
  */
 int libp2p_peerstore_add_peer(struct Peerstore* peerstore, struct Libp2pPeer* peer) {
-	// first check to see if it exists. If it does, return TRUE
+	int retVal = 0;
-	if (libp2p_peerstore_get_peer_entry(peerstore, peer->id, peer->id_size) != NULL)
-		return 1;
 
+	char* ma_string = "";
+	if (peer != NULL && peer->addr_head != NULL && peer->addr_head->item != NULL) {
+		ma_string = ((struct MultiAddress*)peer->addr_head->item)->string;
+	}
+	// first check to see if it exists. If it does, return TRUE
+	if (libp2p_peerstore_get_peer_entry(peerstore, peer->id, peer->id_size) != NULL) {
+		libp2p_logger_debug("peerstore", "Attempted to add %s to peerstore, but already there.\n", ma_string);
+		return 1;
+	}
+
+	if (peer->id_size > 0) {
 		char peer_id[peer->id_size + 1];
 		memcpy(peer_id, peer->id, peer->id_size);
 		peer_id[peer->id_size] = 0;
+		if (peer->addr_head != NULL) {
 			char* address = ((struct MultiAddress*)peer->addr_head->item)->string;
 			libp2p_logger_debug("peerstore", "Adding peer %s with address %s to peer store\n", peer_id, address);
+		}
 		struct PeerEntry* peer_entry = libp2p_peer_entry_new();
 		if (peer_entry == NULL) {
 			return 0;
@@ -112,8 +124,10 @@ int libp2p_peerstore_add_peer(struct Peerstore* peerstore, struct Libp2pPeer* pe
 		peer_entry->peer = libp2p_peer_copy(peer);
 		if (peer_entry->peer == NULL)
 			return 0;
-	int retVal = libp2p_peerstore_add_peer_entry(peerstore, peer_entry);
+		retVal = libp2p_peerstore_add_peer_entry(peerstore, peer_entry);
 		libp2p_peer_entry_free(peer_entry);
+		libp2p_logger_debug("peerstore", "Adding peer %s to peerstore was a success\n", peer_id);
+	}
 	return retVal;
 }
 
@@ -151,3 +165,21 @@ struct Libp2pPeer* libp2p_peerstore_get_peer(struct Peerstore* peerstore, const
 		return NULL;
 	return entry->peer;
 }
+
+/**
+ * Look for this peer in the peerstore. If it is found, return a reference to that object.
+ * If it is not found, add it, and return a reference to the new copy
+ * @param peerstore the peerstore to search
+ * @param in the peer to search for
+ */
+struct Libp2pPeer* libp2p_peerstore_get_or_add_peer(struct Peerstore* peerstore, struct Libp2pPeer* in) {
+	struct Libp2pPeer* out = libp2p_peerstore_get_peer(peerstore, (unsigned char*)in->id, in->id_size);
+	if (out != NULL)
+		return out;
+
+	// we didn't find it. attempt to add
+	if (!libp2p_peerstore_add_peer(peerstore, in))
+		return NULL;
+
+	return libp2p_peerstore_get_peer(peerstore, (unsigned char*)in->id, in->id_size);
+}

peer/providerstore.c

@@ -80,7 +80,7 @@ int libp2p_providerstore_add(struct ProviderStore* store, unsigned char* hash, i
 	return 1;
 }
 
-int libp2p_providerstore_get(struct ProviderStore* store, unsigned char* hash, int hash_size, unsigned char** peer_id, int *peer_id_size) {
+int libp2p_providerstore_get(struct ProviderStore* store, const unsigned char* hash, int hash_size, unsigned char** peer_id, int *peer_id_size) {
 	struct ProviderEntry* current = NULL;
 	for (int i = 0; i < store->provider_entries->total; i++) {
 		current = (struct ProviderEntry*)libp2p_utils_vector_get(store->provider_entries, i);

routing/dht_protocol.c

@@ -5,6 +5,7 @@
 #include "libp2p/routing/dht_protocol.h"
 #include "libp2p/record/message.h"
 #include "libp2p/utils/logger.h"
+#include "libp2p/conn/session.h"
 
 
 /***
@@ -97,12 +98,15 @@ int libp2p_routing_dht_handle_get_providers(struct SessionContext* session, stru
 
 	// Can I provide it?
 	if (libp2p_providerstore_get(providerstore, message->key, message->key_size, &peer_id, &peer_id_size)) {
+		libp2p_logger_debug("dht_protocol", "I can provide a provider for this key.\n");
 		// we have a peer id, convert it to a peer object
 		struct Libp2pPeer* peer = libp2p_peerstore_get_peer(peerstore, peer_id, peer_id_size);
 		if (peer != NULL) {
 			message->provider_peer_head = libp2p_utils_linked_list_new();
-			message->provider_peer_head->item = peer;
+			message->provider_peer_head->item = libp2p_peer_copy(peer);
 		}
+	} else {
+		libp2p_logger_debug("dht_protocol", "I cannot provide a provider for this key.\n");
 	}
 	free(peer_id);
 	// TODO: find closer peers
@@ -113,8 +117,10 @@ int libp2p_routing_dht_handle_get_providers(struct SessionContext* session, stru
 	}
 	*/
 	if (message->provider_peer_head != NULL) {
+		libp2p_logger_debug("dht_protocol", "GetProviders: We have a peer. Sending it back\n");
 		// protobuf it and send it back
 		if (!libp2p_routing_dht_protobuf_message(message, results, results_size)) {
+			libp2p_logger_error("dht_protocol", "GetProviders: Error protobufing results\n");
 			return 0;
 		}
 	}
@@ -162,6 +168,7 @@ int libp2p_routing_dht_handle_add_provider(struct SessionContext* session, struc
 	if (message->record != NULL && message->record->author != NULL && message->record->author_size > 0
 			&& message->key != NULL && message->key_size > 0)
 	*/
+
 	struct Libp2pLinkedList* current = message->provider_peer_head;
 	if (current == NULL) {
 		libp2p_logger_error("dht_protocol", "Provider has no peer.\n");
@@ -199,7 +206,7 @@ int libp2p_routing_dht_handle_add_provider(struct SessionContext* session, struc
 		new_head->next = peer->addr_head;
 		peer->addr_head = new_head;
 		// now add the peer to the peerstore
-		libp2p_logger_debug("dht_protocol", "About to add peer to peerstore\n");
+		libp2p_logger_debug("dht_protocol", "About to add peer %s to peerstore\n", peer_ma->string);
 		if (!libp2p_peerstore_add_peer(peerstore, peer))
 			goto exit;
 		libp2p_logger_debug("dht_protocol", "About to add key to providerstore\n");
@@ -240,14 +247,19 @@ int libp2p_routing_dht_handle_add_provider(struct SessionContext* session, struc
  */
 int libp2p_routing_dht_handle_get_value(struct SessionContext* session, struct Libp2pMessage* message,
 		struct Peerstore* peerstore, struct ProviderStore* providerstore, unsigned char** result_buffer, size_t *result_buffer_size) {
-	//TODO: implement this
+
 	struct Datastore* datastore = session->datastore;
-	size_t data_size = 65535;
+	struct Filestore* filestore = session->filestore;
-	char* data = malloc(data_size);
+	size_t data_size = 0;
-	if (!datastore->datastore_get(message->key, message->key_size, data, data_size, &data_size, datastore)) {
+	unsigned char* data = NULL;
-		free(data);
+
-		return 0;
+	// We need to get the data from the disk
+	if(!filestore->node_get(message->key, message->key_size, (void**)&data, &data_size, filestore)) {
+		libp2p_logger_debug("dht_protocol", "handle_get_value: Unable to get key from filestore\n");
 	}
+
+	libp2p_logger_debug("dht_protocol", "handle_get_value: value retrieved from the datastore\n");
+
 	struct Libp2pRecord *record = libp2p_record_new();
 	record->key_size = message->key_size;
 	record->key = malloc(record->key_size);
@@ -352,11 +364,11 @@ int libp2p_routing_dht_handle_message(struct SessionContext* session, struct Pee
 	}
 	// if we have something to send, send it.
 	if (result_buffer != NULL) {
-		libp2p_logger_debug("dht_protocol", "Sending message back to caller\n");
+		libp2p_logger_debug("dht_protocol", "Sending message back to caller. Message type: %d\n", message->message_type);
 		if (!session->default_stream->write(session, result_buffer, result_buffer_size))
 			goto exit;
 	} else {
-		libp2p_logger_debug("dht_protocol", "Nothing to send back. Kademlia call has been handled\n");
+		libp2p_logger_debug("dht_protocol", "DhtHandleMessage: Nothing to send back. Kademlia call has been handled. Message type: %d\n", message->message_type);
 	}
 	retVal = 1;
 	exit:

utils/logger.c

@@ -17,9 +17,6 @@ struct Libp2pVector* logger_classes = NULL;
  */
 void libp2p_logger_init() {
 	logger_classes = libp2p_utils_vector_new(1);
-	libp2p_logger_add_class("secio");
-	libp2p_logger_add_class("null");
-	libp2p_logger_add_class("dht_protocol");
 }
 
 /***
@@ -88,14 +85,20 @@ void libp2p_logger_log(const char* area, int log_level, const char* format, ...)
 void libp2p_logger_vlog(const char* area, int log_level, const char* format, va_list argptr) {
 	if (!libp2p_logger_initialized())
 		libp2p_logger_init();
+	// only allow a message if the message log level is less than the current loglevel
 	if (log_level <= CURRENT_LOGLEVEL) {
 		int found = 0;
+		// error should always be printed for now. We need to think about this more...
+		if (log_level <= LOGLEVEL_ERROR )
+			found = 1;
+		else {
 			for (int i = 0; i < logger_classes->total; i++) {
 				if (strcmp(libp2p_utils_vector_get(logger_classes, i), area) == 0) {
 					found = 1;
 					break;
 				}
 			}
+		}
 		if (found) {
 			vfprintf(stderr, format, argptr);
 		}