Beginnings of persistence for unixfs

include/ipfs/unixfs/unixfs.h

@@ -0,0 +1,95 @@
+/***
+ * A unix-like file system over IPFS blocks
+ */
+
+#pragma once
+
+/**
+ * The protobuf info:
+ * message Data {
+ *  enum DataType {
+ *		Raw = 0;
+ *		Directory = 1;
+ *		File = 2;
+ *		Metadata = 3;
+ *		Symlink = 4;
+ *	}
+ *
+ *	required DataType Type = 1;
+ *	optional bytes Data = 2;
+ *	optional uint64 filesize = 3;
+ *	repeated uint64 blocksizes = 4;
+ * }
+ *
+ * message Metadata {
+ *	optional string MimeType = 1;
+ * }
+ */
+
+enum UnixFSDataType {
+	UNIXFS_RAW,
+	UNIXFS_DIRECTORY,
+	UNIXFS_FILE,
+	UNIXFS_METADATA,
+	UNIXFS_SYMLINK
+};
+
+struct UnixFSBlockSizeNode {
+	size_t block_size;
+	struct UnixFSBlockSizeNode* next;
+};
+
+struct UnixFS {
+	enum UnixFSDataType data_type;
+	unsigned char* bytes; // an array of bytes
+	size_t bytes_size; // the size of the bytes array
+	size_t file_size; // the file size
+	struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes
+};
+
+struct UnixFSMetaData {
+	char* mime_type;
+};
+
+/**
+ * Allocate memory for a new UnixFS struct
+ * @param obj the pointer to the new object
+ * @returns true(1) on success
+ */
+int ipfs_unixfs_new(struct UnixFS** obj);
+
+/***
+ * Free the resources used by a UnixFS struct
+ * @param obj the struct to free
+ * @returns true(1)
+ */
+int ipfs_unixfs_free(struct UnixFS* obj);
+
+/**
+ * Protobuf functions
+ */
+
+/**
+ * Calculate the max size of the protobuf before encoding
+ * @param obj what will be encoded
+ * @returns the size of the buffer necessary to encode the object
+ */
+size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj);
+
+/***
+ * Encode a UnixFS object into protobuf format
+ * @param incoming the incoming object
+ * @param outgoing where the bytes will be placed
+ * @param max_buffer_size the size of the outgoing buffer
+ * @param bytes_written how many bytes were written in the buffer
+ * @returns true(1) on success
+ */
+int ipfs_unixfs_protobuf_encode(struct UnixFS* incoming, unsigned char* outgoing, size_t max_buffer_size, size_t* bytes_written);
+
+/***
+ * Decodes a protobuf array of bytes into a UnixFS object
+ * @param incoming the array of bytes
+ * @param incoming_size the length of the array
+ * @param outgoing the UnixFS object
+ */
+int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, struct UnixFS** outgoing);

test/Makefile

@@ -20,6 +20,7 @@ OBJS = testit.o test_helper.o \
 	../repo/config/bootstrap_peers.o ../repo/config/datastore.o ../repo/config/gateway.o \
 	../repo/config/addresses.o ../repo/config/swarm.o ../repo/config/peer.o \
 	../thirdparty/ipfsaddr/ipfs_addr.o \
+	../unixfs/unixfs.o \
 	../../c-protobuf/protobuf.o ../../c-protobuf/varint.o
 
 %.o: %.c $(DEPS)

test/storage/test_unixfs.h

@@ -0,0 +1,38 @@
+#include "ipfs/unixfs/unixfs.h"
+
+int test_unixfs_encode_decode() {
+	struct UnixFS* unixfs = NULL;
+	int retVal;
+
+	// a directory
+	retVal = ipfs_unixfs_new(&unixfs);
+	unixfs->data_type = UNIXFS_DIRECTORY;
+
+	// serialize
+	size_t buffer_size = ipfs_unixfs_protobuf_encode_size(unixfs);
+	unsigned char buffer[buffer_size];
+	size_t bytes_written = 0;
+
+	retVal = ipfs_unixfs_protobuf_encode(unixfs, buffer, buffer_size, &bytes_written);
+	if (retVal == 0) {
+		return 0;
+	}
+
+	// unserialize
+	struct UnixFS* results = NULL;
+	retVal = ipfs_unixfs_protobuf_decode(buffer, bytes_written, &results);
+	if (retVal == 0) {
+		return 0;
+	}
+
+	// compare
+	if (results->data_type != unixfs->data_type) {
+		return 0;
+	}
+
+	if (results->block_size_head != unixfs->block_size_head) {
+		return 0;
+	}
+
+	return 1;
+}

test/testit.c

@@ -11,6 +11,7 @@
 #include "storage/test_ds_helper.h"
 #include "storage/test_datastore.h"
 #include "storage/test_blocks.h"
+#include "storage/test_unixfs.h"
  		 
 int testit(const char* name, int (*func)(void)) {
 	printf("Testing %s...\n", name);
@@ -50,7 +51,8 @@ const char* names[] = {
 		"test_merkledag_add_data",
 		"test_merkledag_get_data",
 		"test_merkledag_add_node",
-		"test_merkledag_add_node_with_links"
+		"test_merkledag_add_node_with_links",
+		"test_unixfs_encode_decode"
 };
 
 int (*funcs[])(void) = {
@@ -81,7 +83,8 @@ int (*funcs[])(void) = {
 		test_merkledag_add_data,
 		test_merkledag_get_data,
 		test_merkledag_add_node,
-		test_merkledag_add_node_with_links
+		test_merkledag_add_node_with_links,
+		test_unixfs_encode_decode
 };
 
 /**

unixfs/unixfs.c

@@ -0,0 +1,196 @@
+/***
+ * A unix-like file system over IPFS blocks
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "ipfs/unixfs/unixfs.h"
+#include "protobuf.h"
+#include "varint.h"
+
+/**
+ * The protobuf info:
+ * message Data {
+ *  enum DataType {
+ *		Raw = 0;
+ *		Directory = 1;
+ *		File = 2;
+ *		Metadata = 3;
+ *		Symlink = 4;
+ *	}
+ *
+ *	required DataType Type = 1;
+ *	optional bytes Data = 2;
+ *	optional uint64 filesize = 3;
+ *	repeated uint64 blocksizes = 4;
+ * }
+ *
+ * message Metadata {
+ *	optional string MimeType = 1;
+ * }
+ */
+
+/**
+ * Allocate memory for a new UnixFS struct
+ * @param obj the pointer to the new object
+ * @returns true(1) on success
+ */
+int ipfs_unixfs_new(struct UnixFS** obj) {
+	*obj = (struct UnixFS*)malloc(sizeof(struct UnixFS));
+	if (*obj == NULL)
+		return 0;
+	(*obj)->bytes = 0;
+	(*obj)->bytes_size = 0;
+	(*obj)->data_type = UNIXFS_RAW;
+	(*obj)->file_size = 0;
+	(*obj)->block_size_head = NULL;
+	return 1;
+}
+
+int ipfs_unixfs_free(struct UnixFS* obj) {
+	if (obj != NULL) {
+		free(obj);
+		obj = NULL;
+	}
+	return 1;
+}
+
+/**
+ * Protobuf functions
+ */
+
+//                                            data type         bytes                    file size           block sizes
+enum WireType ipfs_unixfs_message_fields[] = { WIRETYPE_VARINT, WIRETYPE_LENGTH_DELIMITED, WIRETYPE_VARINT, WIRETYPE_VARINT };
+
+/**
+ * Calculate the max size of the protobuf before encoding
+ * @param obj what will be encoded
+ * @returns the size of the buffer necessary to encode the object
+ */
+size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj) {
+	size_t sz = 0;
+	// bytes
+	sz += obj->bytes_size + 11;
+	// data tupe
+	sz += 2;
+	// file_size
+	sz += 11;
+	// block sizes
+	struct UnixFSBlockSizeNode* currNode = obj->block_size_head;
+	while(currNode != NULL) {
+		sz += 11;
+		currNode = currNode->next;
+	}
+	return sz;
+}
+
+/***
+ * Encode a UnixFS object into protobuf format
+ * @param incoming the incoming object
+ * @param outgoing where the bytes will be placed
+ * @param max_buffer_size the size of the outgoing buffer
+ * @param bytes_written how many bytes were written in the buffer
+ * @returns true(1) on success
+ */
+int ipfs_unixfs_protobuf_encode(struct UnixFS* incoming, unsigned char* outgoing, size_t max_buffer_size, size_t* bytes_written) {
+	size_t bytes_used = 0;
+	*bytes_written = 0;
+	int retVal = 0;
+	if (incoming != NULL) {
+		// data type (required)
+		retVal = protobuf_encode_varint(1, ipfs_unixfs_message_fields[0], incoming->data_type, outgoing, max_buffer_size - *bytes_written, &bytes_used);
+		if (retVal == 0)
+			return 0;
+		*bytes_written += bytes_used;
+		// bytes (optional)
+		if (incoming->bytes_size > 0) {
+			retVal = protobuf_encode_length_delimited(2, ipfs_unixfs_message_fields[1], (char*)incoming->bytes, incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
+			if (retVal == 0)
+				return 0;
+			*bytes_written += bytes_used;
+		}
+		// file size (optional)
+		if (incoming->file_size > 0) {
+			retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
+			if (retVal == 0)
+				return 0;
+			*bytes_written += bytes_used;
+		}
+		// block sizes
+		struct UnixFSBlockSizeNode* currNode = incoming->block_size_head;
+		while (currNode != NULL) {
+			retVal = protobuf_encode_varint(4, ipfs_unixfs_message_fields[3], currNode->block_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
+			currNode = currNode->next;
+		}
+	}
+	return 1;
+}
+
+/***
+ * Decodes a protobuf array of bytes into a UnixFS object
+ * @param incoming the array of bytes
+ * @param incoming_size the length of the array
+ * @param outgoing the UnixFS object
+ */
+int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, struct UnixFS** outgoing) {
+	// short cut for nulls
+	if (incoming_size == 0) {
+		*outgoing = NULL;
+		return 1;
+	}
+
+	size_t pos = 0;
+	int retVal = 0;
+
+	if (ipfs_unixfs_new(outgoing) == 0) {
+		return 0;
+	}
+	struct UnixFS* result = *outgoing;
+
+	while(pos < incoming_size) {
+		size_t bytes_read = 0;
+		int field_no;
+		enum WireType field_type;
+		if (protobuf_decode_field_and_type(&incoming[pos], incoming_size, &field_no, &field_type, &bytes_read) == 0) {
+			return 0;
+		}
+		pos += bytes_read;
+		switch(field_no) {
+			case (1): // data type (varint)
+				result->data_type = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
+				pos += bytes_read;
+				break;
+			case (2): // bytes (length delimited)
+				retVal = protobuf_decode_length_delimited(&incoming[pos], incoming_size - pos, (char**)&(result->bytes), &(result->bytes_size), &bytes_read);
+				if (retVal == 0)
+					return 0;
+				pos += bytes_read;
+				break;
+			case (3): // file size
+				result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
+				pos += bytes_read;
+				break;
+			case (4): { // block sizes (linked list from varint)
+				struct UnixFSBlockSizeNode* currNode = (struct UnixFSBlockSizeNode*)malloc(sizeof(struct UnixFSBlockSizeNode));
+				if (currNode == NULL)
+					return 0;
+				currNode->next = NULL;
+				currNode->block_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
+				pos += bytes_read;
+				if (result->block_size_head == NULL) {
+					result->block_size_head = currNode;
+				} else {
+					struct UnixFSBlockSizeNode* last = result->block_size_head;
+					while (last->next != NULL)
+						last = last->next;
+					last->next = currNode;
+				}
+				break;
+			}
+		}
+
+	}
+
+	return 1;
+}