Moved datastore interface from ipfs to libp2p

Makefile

@@ -6,10 +6,12 @@ OBJS = \
 	conn/*.o \
 	crypto/*.o \
 	crypto/encoding/*.o \
+	db/datastore.o \
 	thirdparty/mbedtls/*.o \
 	hashmap/hashmap.o \
 	net/*.o \
 	nodeio/*.o \
+	os/*.o \
 	peer/*.o \
 	record/*.o \
 	routing/*.o \
@@ -22,10 +24,12 @@ link:
 compile:
 	cd conn; make all;
 	cd crypto; make all;
+	cd db; make all;
 	cd thirdparty; make all;
 	cd hashmap; make all;
 	cd net; make all;
 	cd nodeio; make all;
+	cd os; make all;
 	cd peer; make all;
 	cd record; make all;
 	cd routing; make all;
@@ -42,9 +46,11 @@ all: test
 clean:
 	cd conn; make clean;
 	cd crypto; make clean;
+	cd db; make clean;
 	cd hashmap; make clean;
 	cd net; make clean;
 	cd nodeio; make clean;
+	cd os; make clean;
 	cd peer; make clean;
 	cd thirdparty; make clean
 	cd record; make clean;

db/Makefile

@@ -0,0 +1,14 @@
+CC = gcc
+CFLAGS = -O0 -I../include -I../../c-protobuf -I../../c-multihash/include -I../../c-multiaddr/include -g3 -std=c99
+LFLAGS =
+DEPS = 
+OBJS = datastore.o
+
+%.o: %.c $(DEPS)
+	$(CC) -c -o $@ $< $(CFLAGS)
+
+	
+all: $(OBJS)
+
+clean:
+	rm -f $(OBJS)

db/datastore.c

@@ -0,0 +1,75 @@
+#include <stdlib.h>
+#include <string.h>
+
+#include "libp2p/db/datastore.h"
+#include "libp2p/os/utils.h"
+
+int alloc_and_assign(char** result, const char* string) {
+	*result = malloc(strlen(string)+1);
+	if (*result == NULL)
+		return 0;
+	strcpy(*result, string);
+	return 1;
+}
+
+/***
+ * initialize the structure of the datastore
+ * @param datastore the struct to initialize
+ * @returns true(1) on success
+ */
+int libp2p_datastore_init(struct Datastore* datastore, const char* config_root) {
+	unsigned long stringLength = strlen(config_root) + 12;
+	datastore->path = malloc(sizeof(char) * stringLength);
+	os_utils_filepath_join(config_root, "datastore", datastore->path, stringLength);
+	alloc_and_assign(&datastore->type, "lmdb");
+	alloc_and_assign(&datastore->storage_max, "10GB");
+	datastore->storage_gc_watermark = 90;
+	alloc_and_assign(&datastore->gc_period, "1h");
+	datastore->hash_on_read = 0;
+	datastore->bloom_filter_size = 0;
+	datastore->no_sync = 0;
+	return 1;
+}
+
+/***
+ * initialize the structure of the datastore
+ * @param datastore the struct to initialize
+ * @returns true(1) on success
+ */
+int libp2p_datastore_new(struct Datastore** datastore) {
+	*datastore = malloc(sizeof(struct Datastore));
+	if (*datastore == NULL)
+		return 0;
+	(*datastore)->path = NULL;
+	(*datastore)->handle = NULL;
+	(*datastore)->type = NULL;
+	(*datastore)->storage_max = NULL;
+	(*datastore)->gc_period = NULL;
+	(*datastore)->params = NULL;
+	return 1;
+}
+
+/***
+ * deallocate the memory and clear resources from a datastore_init
+ * @param datastore the struct to deallocate
+ * @returns true(1)
+ */
+int libp2p_datastore_free(struct Datastore* datastore) {
+	if (datastore != NULL)
+	{
+		if (datastore->path != NULL)
+			free(datastore->path);
+		if (datastore->type != NULL)
+			free(datastore->type);
+		if (datastore->storage_max != NULL)
+			free(datastore->storage_max);
+		if (datastore->gc_period != NULL)
+			free(datastore->gc_period);
+		if (datastore->params != NULL)
+			free(datastore->params);
+		if (datastore->handle != NULL)
+			datastore->datastore_close(datastore);
+		free(datastore);
+	}
+	return 1;
+}

include/libp2p/db/datastore.h

@@ -0,0 +1,59 @@
+#pragma once
+
+#include <stdint.h>
+
+/***
+ * Interface to data storage
+ */
+
+enum DatastoreCursorOp { CURSOR_FIRST, CURSOR_NEXT };
+
+struct Datastore {
+	char* type;
+	char* path;
+	char* storage_max;
+	int storage_gc_watermark;
+	char* gc_period;
+	char* params;
+	int no_sync;
+	int hash_on_read;
+	int bloom_filter_size;
+
+	// function pointers for datastore operations
+	int (*datastore_open)(int argc, char** argv, struct Datastore* datastore);
+	int (*datastore_close)(struct Datastore* datastore);
+	int (*datastore_put)(const unsigned char* key, size_t key_size, unsigned char* data, size_t data_length, const struct Datastore* datastore);
+	int (*datastore_get)(const char* key, size_t key_size,
+			unsigned char* data, size_t max_data_length, size_t* data_length,
+			const struct Datastore* datastore);
+	int (*datastore_cursor_open)(struct Datastore* datastore);
+	int (*datastore_cursor_close)(struct Datastore* datastore);
+	int (*datastore_cursor_get)(unsigned char** key, int* key_length, unsigned char** value, int* value_length, enum DatastoreCursorOp op, struct Datastore* datastore);
+	// generic connection and status variables for the datastore
+	void* handle; // a handle to the database
+	void* cursor; // a current cursor
+};
+
+/***
+ * Initialize the structure of the datastore to default settings. Used for
+ * creating a new datastore on the disk.
+ * @param datastore the struct to initialize
+ * @param config_root the path to the root of IPFS
+ * @returns true(1) on success
+ */
+int libp2p_datastore_init(struct Datastore* datastore, const char* config_root);
+
+/***
+ * initialize the structure of the datastore
+ * @param datastore the struct to initialize
+ * @returns true(1) on success
+ */
+int libp2p_datastore_new(struct Datastore** datastore);
+
+
+/***
+ * deallocate the memory and clear resources from a datastore_init
+ * @param datastore the struct to deallocate
+ * @returns true(1)
+ */
+int libp2p_datastore_free(struct Datastore* datastore);

include/libp2p/os/utils.h

@@ -0,0 +1,80 @@
+/**
+ * OS specific stuff. This is going to get ugly, but at least its in one place
+ */
+#ifndef __OS_UTILS_H__
+#define __OS_UTILS_H__
+
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/**
+ * a linked list to store filenames
+ */
+struct FileList {
+	char* file_name;
+	struct FileList* next;
+};
+
+/**
+ * Builds a list of files within a directory
+ * @param path the path to examine
+ * @returns a FileList struct of the first file
+ */
+struct FileList* os_utils_list_directory(const char* path);
+
+/**
+ * Cleans up memory used by a FileList struct
+ * @param first the struct to free
+ * @returns true(1)
+ */
+int os_utils_free_file_list(struct FileList* first);
+
+/**
+ * Split the filename from the path
+ * @param in the full path and filename
+ * @param path only the path part
+ * @param filename only the file name
+ * @returns true(1)
+ */
+int os_utils_split_filename(const char* in, char** path, char** filename);
+
+
+/**
+ * get an environment varible from the os
+ * @param variable the variable to look for
+ * @returns the results
+ */
+char* os_utils_getenv(const char* variable);
+int os_utils_setenv(const char* variable, const char* value, int overwrite);
+/**
+ * get the user's home directory
+ * @returns the user's home directory
+ */
+char* os_utils_get_homedir();
+
+/**
+ * join 2 pieces of a filepath, being careful about the slashes
+ * @param root the root part
+ * @param extension what should be concatenated
+ * @param results where to put the results
+ * @param max_len throw an error if the total is longer than max_len
+ */
+int os_utils_filepath_join(const char* root, const char* extension, char* results, unsigned long max_len);
+
+int os_utils_file_exists(const char* file_name);
+
+int os_utils_file_size(const char* file_name);
+
+int os_utils_directory_writeable(const char* path);
+
+int os_utils_directory_exists(const char* path);
+
+/**
+ * Determine if the path presented is actually a directory
+ * @param file_name the path to examine
+ * @returns true(1) if file_name is actually a directory
+ */
+int os_utils_is_directory(const char* file_name);
+
+#endif /* utils_h */

os/Makefile

@@ -0,0 +1,14 @@
+CC = gcc
+CFLAGS = -O0 -I../include -I../../c-protobuf -I../../c-multihash/include -I../../c-multiaddr/include -g3
+LFLAGS =
+DEPS = 
+OBJS = utils.o
+
+%.o: %.c $(DEPS)
+	$(CC) -c -o $@ $< $(CFLAGS)
+
+	
+all: $(OBJS)
+
+clean:
+	rm -f $(OBJS)

os/utils.c

@@ -0,0 +1,172 @@
+#include "libp2p/os/utils.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#ifndef __MINGW32__
+	#include <pwd.h>
+#endif
+#include <unistd.h>
+#include <stdio.h>
+#include <dirent.h>
+
+/**
+ * get an environment varible from the os
+ * @param variable the variable to look for
+ * @returns the results
+ */
+char* os_utils_getenv(const char* variable) {
+	return getenv(variable);
+}
+
+/**
+ * set an environment variable in the os
+ * @param variable the variable to set
+ * @param value the value to assign to the variable
+ * @param overwrite passing a non-zero will allow an existing value to be overwritten
+ * @returns true(1) on success
+ */
+int os_utils_setenv(const char* variable, const char* value, int overwrite) {
+#ifdef __MINGW32__
+	return 0;
+#else
+	setenv(variable, value, overwrite);
+	return 1;
+#endif
+}
+
+/**
+ * returns the user's home directory
+ * @returns the home directory
+ */
+char* os_utils_get_homedir() {
+#ifndef __MINGW32__
+	struct passwd *pw = getpwuid(getuid());
+	return pw->pw_dir;
+#endif
+	return ".";
+}
+
+/**
+ * join 2 pieces of a filepath, being careful about the slashes
+ * @param root the root part
+ * @param extension what should be concatenated
+ * @param results where to put the results
+ * @param max_len throw an error if the total is longer than max_len
+ */
+int os_utils_filepath_join(const char* root, const char* extension, char* results, unsigned long max_len) {
+	if (strlen(root) + strlen(extension) + 1 > max_len)
+		return 0;
+	strncpy(results, root, strlen(root) + 1);
+	// one of these should have a slash. If not, add one
+	if (root[strlen(root)-1] != '/' && extension[0] != '/') {
+		results[strlen(root)] = '/';
+		results[strlen(root)+1] = 0;
+	}
+	strncat(results, extension, strlen(extension)+1);
+	return 1;
+}
+
+int os_utils_file_exists(const char* file_name) {
+	if (access(file_name, F_OK) != -1)
+		return 1;
+	return 0;
+}
+
+int os_utils_directory_exists(const char* directory_name) {
+	if (access(directory_name, F_OK) != -1)
+		return 1;
+	return 0;
+}
+
+int os_utils_is_directory(const char* file_name) {
+	struct stat path_stat;
+	stat(file_name, &path_stat);
+	return S_ISDIR(path_stat.st_mode) != 0;
+}
+
+int os_utils_split_filename(const char* in, char** path, char** filename) {
+	int len = strlen(in);
+	char* pos = strrchr(in, '/');
+	if (pos != NULL) {
+		pos++;
+		*path = (char*)malloc((pos - in) + 1);
+		strncpy(*path, in, pos-in-1);
+		(*path)[pos-in-1] = 0;
+		*filename = (char*)malloc(len - (pos-in) + 1);
+		strcpy(*filename, pos);
+		(*filename)[len - (pos-in)] = 0;
+	} else {
+		*filename = (char*)malloc(len+1);
+		strcpy(*filename, in);
+	}
+	return 1;
+}
+
+struct FileList* os_utils_list_directory(const char* path) {
+	DIR* dp;
+	struct dirent *ep;
+	struct FileList* first = NULL;
+	struct FileList* last = NULL;
+	struct FileList* next = NULL;
+
+	dp = opendir(path);
+	if (dp != NULL) {
+		while ( (ep = readdir(dp)) ) {
+			if (strcmp(ep->d_name, ".") != 0 && strcmp(ep->d_name, "..") != 0) {
+				// grab the data
+				next = (struct FileList*)malloc(sizeof(struct FileList));
+				next->file_name = malloc(strlen(ep->d_name) + 1);
+				strcpy(next->file_name, ep->d_name);
+				next->next = NULL;
+				// put it in the appropriate spot
+				if (first == NULL) {
+					first = next;
+					last = next;
+				} else {
+					last->next = next;
+					last = next;
+				}
+			} // not dealing with . or ..
+		} // while
+		closedir(dp);
+	}
+	return first;
+}
+
+int os_utils_free_file_list(struct FileList* first) {
+	if (first != NULL) {
+		struct FileList* next = first;
+		struct FileList* last = NULL;
+		while (next != NULL) {
+			last = next->next;
+			if (next->file_name != NULL) {
+				free(next->file_name);
+			}
+			free(next);
+			next = last;
+		}
+	}
+	return 1;
+}
+
+
+
+int os_utils_directory_writeable(const char* path) {
+	int result = access(path, W_OK);
+	return result == 0;
+}
+
+int os_utils_file_size(const char* path) {
+	size_t file_size = 0;
+	// open file
+	FILE* in_file = fopen(path, "r");
+	if (in_file == NULL)
+		return 0;
+	// determine size
+	fseek(in_file, 0L, SEEK_END);
+	file_size = ftell(in_file);
+	fclose(in_file);
+	return file_size;
+}

peer/providerstore.c

@@ -50,7 +50,7 @@ int libp2p_providerstore_add(struct ProviderStore* store, unsigned char* hash, i
 	char peer_str[peer_id_size + 1];
 	memcpy(peer_str, peer_id, peer_id_size);
 	peer_str[peer_id_size] = 0;
-	libp2p_logger_debug("providerstore", "Adding hash %s to providerstore. It can be retrieved from %s", hash_str, peer_str);
+	libp2p_logger_debug("providerstore", "Adding hash %s to providerstore. It can be retrieved from %s\n", hash_str, peer_str);
 	struct ProviderEntry* entry = (struct ProviderEntry*)malloc(sizeof(struct ProviderEntry));
 	entry->hash = malloc(hash_size);
 	memcpy(entry->hash, hash, hash_size);

routing/dht_protocol.c

@@ -169,6 +169,7 @@ int libp2p_routing_dht_handle_add_provider(struct SessionContext* session, struc
 		sprintf(new_string, "/ip4/%s/tcp/%d", ip, port);
 		struct MultiAddress* new_ma = multiaddress_new_from_string(new_string);
 		libp2p_logger_debug("dht_protocol", "New MultiAddress made with %s.\n", new_string);
+		// TODO: See if the sender is who he says he is
 		// set it as the first in the list
 		struct Libp2pLinkedList* new_head = libp2p_utils_linked_list_new();
 		new_head->item = new_ma;