From 57ed4fd5e4e174f9657a1ba5028f96b85e987ee5 Mon Sep 17 00:00:00 2001 From: jmjatlanta Date: Mon, 19 Dec 2016 14:19:43 -0500 Subject: [PATCH] Beginnings of persistence for unixfs --- include/ipfs/unixfs/unixfs.h | 95 +++++++++++++++++ test/Makefile | 1 + test/storage/test_unixfs.h | 38 +++++++ test/testit.c | 7 +- unixfs/unixfs.c | 196 +++++++++++++++++++++++++++++++++++ 5 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 include/ipfs/unixfs/unixfs.h create mode 100644 test/storage/test_unixfs.h create mode 100644 unixfs/unixfs.c diff --git a/include/ipfs/unixfs/unixfs.h b/include/ipfs/unixfs/unixfs.h new file mode 100644 index 0000000..f279710 --- /dev/null +++ b/include/ipfs/unixfs/unixfs.h @@ -0,0 +1,95 @@ +/*** + * A unix-like file system over IPFS blocks + */ + +#pragma once + +/** + * The protobuf info: + * message Data { + * enum DataType { + * Raw = 0; + * Directory = 1; + * File = 2; + * Metadata = 3; + * Symlink = 4; + * } + * + * required DataType Type = 1; + * optional bytes Data = 2; + * optional uint64 filesize = 3; + * repeated uint64 blocksizes = 4; + * } + * + * message Metadata { + * optional string MimeType = 1; + * } + */ + +enum UnixFSDataType { + UNIXFS_RAW, + UNIXFS_DIRECTORY, + UNIXFS_FILE, + UNIXFS_METADATA, + UNIXFS_SYMLINK +}; + +struct UnixFSBlockSizeNode { + size_t block_size; + struct UnixFSBlockSizeNode* next; +}; + +struct UnixFS { + enum UnixFSDataType data_type; + unsigned char* bytes; // an array of bytes + size_t bytes_size; // the size of the bytes array + size_t file_size; // the file size + struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes +}; + +struct UnixFSMetaData { + char* mime_type; +}; + +/** + * Allocate memory for a new UnixFS struct + * @param obj the pointer to the new object + * @returns true(1) on success + */ +int ipfs_unixfs_new(struct UnixFS** obj); + +/*** + * Free the resources used by a UnixFS struct + * @param obj the struct to free + * @returns true(1) + */ +int ipfs_unixfs_free(struct UnixFS* obj); + +/** + * Protobuf functions + */ + +/** + * Calculate the max size of the protobuf before encoding + * @param obj what will be encoded + * @returns the size of the buffer necessary to encode the object + */ +size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj); + +/*** + * Encode a UnixFS object into protobuf format + * @param incoming the incoming object + * @param outgoing where the bytes will be placed + * @param max_buffer_size the size of the outgoing buffer + * @param bytes_written how many bytes were written in the buffer + * @returns true(1) on success + */ +int ipfs_unixfs_protobuf_encode(struct UnixFS* incoming, unsigned char* outgoing, size_t max_buffer_size, size_t* bytes_written); + +/*** + * Decodes a protobuf array of bytes into a UnixFS object + * @param incoming the array of bytes + * @param incoming_size the length of the array + * @param outgoing the UnixFS object + */ +int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, struct UnixFS** outgoing); diff --git a/test/Makefile b/test/Makefile index c5142ad..b725c00 100644 --- a/test/Makefile +++ b/test/Makefile @@ -20,6 +20,7 @@ OBJS = testit.o test_helper.o \ ../repo/config/bootstrap_peers.o ../repo/config/datastore.o ../repo/config/gateway.o \ ../repo/config/addresses.o ../repo/config/swarm.o ../repo/config/peer.o \ ../thirdparty/ipfsaddr/ipfs_addr.o \ + ../unixfs/unixfs.o \ ../../c-protobuf/protobuf.o ../../c-protobuf/varint.o %.o: %.c $(DEPS) diff --git a/test/storage/test_unixfs.h b/test/storage/test_unixfs.h new file mode 100644 index 0000000..cc81754 --- /dev/null +++ b/test/storage/test_unixfs.h @@ -0,0 +1,38 @@ +#include "ipfs/unixfs/unixfs.h" + +int test_unixfs_encode_decode() { + struct UnixFS* unixfs = NULL; + int retVal; + + // a directory + retVal = ipfs_unixfs_new(&unixfs); + unixfs->data_type = UNIXFS_DIRECTORY; + + // serialize + size_t buffer_size = ipfs_unixfs_protobuf_encode_size(unixfs); + unsigned char buffer[buffer_size]; + size_t bytes_written = 0; + + retVal = ipfs_unixfs_protobuf_encode(unixfs, buffer, buffer_size, &bytes_written); + if (retVal == 0) { + return 0; + } + + // unserialize + struct UnixFS* results = NULL; + retVal = ipfs_unixfs_protobuf_decode(buffer, bytes_written, &results); + if (retVal == 0) { + return 0; + } + + // compare + if (results->data_type != unixfs->data_type) { + return 0; + } + + if (results->block_size_head != unixfs->block_size_head) { + return 0; + } + + return 1; +} diff --git a/test/testit.c b/test/testit.c index 0c00512..f96843d 100644 --- a/test/testit.c +++ b/test/testit.c @@ -11,6 +11,7 @@ #include "storage/test_ds_helper.h" #include "storage/test_datastore.h" #include "storage/test_blocks.h" +#include "storage/test_unixfs.h" int testit(const char* name, int (*func)(void)) { printf("Testing %s...\n", name); @@ -50,7 +51,8 @@ const char* names[] = { "test_merkledag_add_data", "test_merkledag_get_data", "test_merkledag_add_node", - "test_merkledag_add_node_with_links" + "test_merkledag_add_node_with_links", + "test_unixfs_encode_decode" }; int (*funcs[])(void) = { @@ -81,7 +83,8 @@ int (*funcs[])(void) = { test_merkledag_add_data, test_merkledag_get_data, test_merkledag_add_node, - test_merkledag_add_node_with_links + test_merkledag_add_node_with_links, + test_unixfs_encode_decode }; /** diff --git a/unixfs/unixfs.c b/unixfs/unixfs.c new file mode 100644 index 0000000..c72d6b3 --- /dev/null +++ b/unixfs/unixfs.c @@ -0,0 +1,196 @@ +/*** + * A unix-like file system over IPFS blocks + */ + +#include +#include + +#include "ipfs/unixfs/unixfs.h" +#include "protobuf.h" +#include "varint.h" + +/** + * The protobuf info: + * message Data { + * enum DataType { + * Raw = 0; + * Directory = 1; + * File = 2; + * Metadata = 3; + * Symlink = 4; + * } + * + * required DataType Type = 1; + * optional bytes Data = 2; + * optional uint64 filesize = 3; + * repeated uint64 blocksizes = 4; + * } + * + * message Metadata { + * optional string MimeType = 1; + * } + */ + +/** + * Allocate memory for a new UnixFS struct + * @param obj the pointer to the new object + * @returns true(1) on success + */ +int ipfs_unixfs_new(struct UnixFS** obj) { + *obj = (struct UnixFS*)malloc(sizeof(struct UnixFS)); + if (*obj == NULL) + return 0; + (*obj)->bytes = 0; + (*obj)->bytes_size = 0; + (*obj)->data_type = UNIXFS_RAW; + (*obj)->file_size = 0; + (*obj)->block_size_head = NULL; + return 1; +} + +int ipfs_unixfs_free(struct UnixFS* obj) { + if (obj != NULL) { + free(obj); + obj = NULL; + } + return 1; +} + +/** + * Protobuf functions + */ + +// data type bytes file size block sizes +enum WireType ipfs_unixfs_message_fields[] = { WIRETYPE_VARINT, WIRETYPE_LENGTH_DELIMITED, WIRETYPE_VARINT, WIRETYPE_VARINT }; + +/** + * Calculate the max size of the protobuf before encoding + * @param obj what will be encoded + * @returns the size of the buffer necessary to encode the object + */ +size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj) { + size_t sz = 0; + // bytes + sz += obj->bytes_size + 11; + // data tupe + sz += 2; + // file_size + sz += 11; + // block sizes + struct UnixFSBlockSizeNode* currNode = obj->block_size_head; + while(currNode != NULL) { + sz += 11; + currNode = currNode->next; + } + return sz; +} + +/*** + * Encode a UnixFS object into protobuf format + * @param incoming the incoming object + * @param outgoing where the bytes will be placed + * @param max_buffer_size the size of the outgoing buffer + * @param bytes_written how many bytes were written in the buffer + * @returns true(1) on success + */ +int ipfs_unixfs_protobuf_encode(struct UnixFS* incoming, unsigned char* outgoing, size_t max_buffer_size, size_t* bytes_written) { + size_t bytes_used = 0; + *bytes_written = 0; + int retVal = 0; + if (incoming != NULL) { + // data type (required) + retVal = protobuf_encode_varint(1, ipfs_unixfs_message_fields[0], incoming->data_type, outgoing, max_buffer_size - *bytes_written, &bytes_used); + if (retVal == 0) + return 0; + *bytes_written += bytes_used; + // bytes (optional) + if (incoming->bytes_size > 0) { + retVal = protobuf_encode_length_delimited(2, ipfs_unixfs_message_fields[1], (char*)incoming->bytes, incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used); + if (retVal == 0) + return 0; + *bytes_written += bytes_used; + } + // file size (optional) + if (incoming->file_size > 0) { + retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used); + if (retVal == 0) + return 0; + *bytes_written += bytes_used; + } + // block sizes + struct UnixFSBlockSizeNode* currNode = incoming->block_size_head; + while (currNode != NULL) { + retVal = protobuf_encode_varint(4, ipfs_unixfs_message_fields[3], currNode->block_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used); + currNode = currNode->next; + } + } + return 1; +} + +/*** + * Decodes a protobuf array of bytes into a UnixFS object + * @param incoming the array of bytes + * @param incoming_size the length of the array + * @param outgoing the UnixFS object + */ +int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, struct UnixFS** outgoing) { + // short cut for nulls + if (incoming_size == 0) { + *outgoing = NULL; + return 1; + } + + size_t pos = 0; + int retVal = 0; + + if (ipfs_unixfs_new(outgoing) == 0) { + return 0; + } + struct UnixFS* result = *outgoing; + + while(pos < incoming_size) { + size_t bytes_read = 0; + int field_no; + enum WireType field_type; + if (protobuf_decode_field_and_type(&incoming[pos], incoming_size, &field_no, &field_type, &bytes_read) == 0) { + return 0; + } + pos += bytes_read; + switch(field_no) { + case (1): // data type (varint) + result->data_type = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read); + pos += bytes_read; + break; + case (2): // bytes (length delimited) + retVal = protobuf_decode_length_delimited(&incoming[pos], incoming_size - pos, (char**)&(result->bytes), &(result->bytes_size), &bytes_read); + if (retVal == 0) + return 0; + pos += bytes_read; + break; + case (3): // file size + result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read); + pos += bytes_read; + break; + case (4): { // block sizes (linked list from varint) + struct UnixFSBlockSizeNode* currNode = (struct UnixFSBlockSizeNode*)malloc(sizeof(struct UnixFSBlockSizeNode)); + if (currNode == NULL) + return 0; + currNode->next = NULL; + currNode->block_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read); + pos += bytes_read; + if (result->block_size_head == NULL) { + result->block_size_head = currNode; + } else { + struct UnixFSBlockSizeNode* last = result->block_size_head; + while (last->next != NULL) + last = last->next; + last->next = currNode; + } + break; + } + } + + } + + return 1; +}