Beginnings of persistence for unixfs

This commit is contained in:
jmjatlanta 2016-12-19 14:19:43 -05:00
parent bece919c40
commit 57ed4fd5e4
5 changed files with 335 additions and 2 deletions

View file

@ -0,0 +1,95 @@
/***
* A unix-like file system over IPFS blocks
*/
#pragma once
/**
* The protobuf info:
* message Data {
* enum DataType {
* Raw = 0;
* Directory = 1;
* File = 2;
* Metadata = 3;
* Symlink = 4;
* }
*
* required DataType Type = 1;
* optional bytes Data = 2;
* optional uint64 filesize = 3;
* repeated uint64 blocksizes = 4;
* }
*
* message Metadata {
* optional string MimeType = 1;
* }
*/
enum UnixFSDataType {
UNIXFS_RAW,
UNIXFS_DIRECTORY,
UNIXFS_FILE,
UNIXFS_METADATA,
UNIXFS_SYMLINK
};
struct UnixFSBlockSizeNode {
size_t block_size;
struct UnixFSBlockSizeNode* next;
};
struct UnixFS {
enum UnixFSDataType data_type;
unsigned char* bytes; // an array of bytes
size_t bytes_size; // the size of the bytes array
size_t file_size; // the file size
struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes
};
struct UnixFSMetaData {
char* mime_type;
};
/**
* Allocate memory for a new UnixFS struct
* @param obj the pointer to the new object
* @returns true(1) on success
*/
int ipfs_unixfs_new(struct UnixFS** obj);
/***
* Free the resources used by a UnixFS struct
* @param obj the struct to free
* @returns true(1)
*/
int ipfs_unixfs_free(struct UnixFS* obj);
/**
* Protobuf functions
*/
/**
* Calculate the max size of the protobuf before encoding
* @param obj what will be encoded
* @returns the size of the buffer necessary to encode the object
*/
size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj);
/***
* Encode a UnixFS object into protobuf format
* @param incoming the incoming object
* @param outgoing where the bytes will be placed
* @param max_buffer_size the size of the outgoing buffer
* @param bytes_written how many bytes were written in the buffer
* @returns true(1) on success
*/
int ipfs_unixfs_protobuf_encode(struct UnixFS* incoming, unsigned char* outgoing, size_t max_buffer_size, size_t* bytes_written);
/***
* Decodes a protobuf array of bytes into a UnixFS object
* @param incoming the array of bytes
* @param incoming_size the length of the array
* @param outgoing the UnixFS object
*/
int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, struct UnixFS** outgoing);

View file

@ -20,6 +20,7 @@ OBJS = testit.o test_helper.o \
../repo/config/bootstrap_peers.o ../repo/config/datastore.o ../repo/config/gateway.o \ ../repo/config/bootstrap_peers.o ../repo/config/datastore.o ../repo/config/gateway.o \
../repo/config/addresses.o ../repo/config/swarm.o ../repo/config/peer.o \ ../repo/config/addresses.o ../repo/config/swarm.o ../repo/config/peer.o \
../thirdparty/ipfsaddr/ipfs_addr.o \ ../thirdparty/ipfsaddr/ipfs_addr.o \
../unixfs/unixfs.o \
../../c-protobuf/protobuf.o ../../c-protobuf/varint.o ../../c-protobuf/protobuf.o ../../c-protobuf/varint.o
%.o: %.c $(DEPS) %.o: %.c $(DEPS)

View file

@ -0,0 +1,38 @@
#include "ipfs/unixfs/unixfs.h"
int test_unixfs_encode_decode() {
struct UnixFS* unixfs = NULL;
int retVal;
// a directory
retVal = ipfs_unixfs_new(&unixfs);
unixfs->data_type = UNIXFS_DIRECTORY;
// serialize
size_t buffer_size = ipfs_unixfs_protobuf_encode_size(unixfs);
unsigned char buffer[buffer_size];
size_t bytes_written = 0;
retVal = ipfs_unixfs_protobuf_encode(unixfs, buffer, buffer_size, &bytes_written);
if (retVal == 0) {
return 0;
}
// unserialize
struct UnixFS* results = NULL;
retVal = ipfs_unixfs_protobuf_decode(buffer, bytes_written, &results);
if (retVal == 0) {
return 0;
}
// compare
if (results->data_type != unixfs->data_type) {
return 0;
}
if (results->block_size_head != unixfs->block_size_head) {
return 0;
}
return 1;
}

View file

@ -11,6 +11,7 @@
#include "storage/test_ds_helper.h" #include "storage/test_ds_helper.h"
#include "storage/test_datastore.h" #include "storage/test_datastore.h"
#include "storage/test_blocks.h" #include "storage/test_blocks.h"
#include "storage/test_unixfs.h"
int testit(const char* name, int (*func)(void)) { int testit(const char* name, int (*func)(void)) {
printf("Testing %s...\n", name); printf("Testing %s...\n", name);
@ -50,7 +51,8 @@ const char* names[] = {
"test_merkledag_add_data", "test_merkledag_add_data",
"test_merkledag_get_data", "test_merkledag_get_data",
"test_merkledag_add_node", "test_merkledag_add_node",
"test_merkledag_add_node_with_links" "test_merkledag_add_node_with_links",
"test_unixfs_encode_decode"
}; };
int (*funcs[])(void) = { int (*funcs[])(void) = {
@ -81,7 +83,8 @@ int (*funcs[])(void) = {
test_merkledag_add_data, test_merkledag_add_data,
test_merkledag_get_data, test_merkledag_get_data,
test_merkledag_add_node, test_merkledag_add_node,
test_merkledag_add_node_with_links test_merkledag_add_node_with_links,
test_unixfs_encode_decode
}; };
/** /**

196
unixfs/unixfs.c Normal file
View file

@ -0,0 +1,196 @@
/***
* A unix-like file system over IPFS blocks
*/
#include <stdio.h>
#include <stdlib.h>
#include "ipfs/unixfs/unixfs.h"
#include "protobuf.h"
#include "varint.h"
/**
* The protobuf info:
* message Data {
* enum DataType {
* Raw = 0;
* Directory = 1;
* File = 2;
* Metadata = 3;
* Symlink = 4;
* }
*
* required DataType Type = 1;
* optional bytes Data = 2;
* optional uint64 filesize = 3;
* repeated uint64 blocksizes = 4;
* }
*
* message Metadata {
* optional string MimeType = 1;
* }
*/
/**
* Allocate memory for a new UnixFS struct
* @param obj the pointer to the new object
* @returns true(1) on success
*/
int ipfs_unixfs_new(struct UnixFS** obj) {
*obj = (struct UnixFS*)malloc(sizeof(struct UnixFS));
if (*obj == NULL)
return 0;
(*obj)->bytes = 0;
(*obj)->bytes_size = 0;
(*obj)->data_type = UNIXFS_RAW;
(*obj)->file_size = 0;
(*obj)->block_size_head = NULL;
return 1;
}
int ipfs_unixfs_free(struct UnixFS* obj) {
if (obj != NULL) {
free(obj);
obj = NULL;
}
return 1;
}
/**
* Protobuf functions
*/
// data type bytes file size block sizes
enum WireType ipfs_unixfs_message_fields[] = { WIRETYPE_VARINT, WIRETYPE_LENGTH_DELIMITED, WIRETYPE_VARINT, WIRETYPE_VARINT };
/**
* Calculate the max size of the protobuf before encoding
* @param obj what will be encoded
* @returns the size of the buffer necessary to encode the object
*/
size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj) {
size_t sz = 0;
// bytes
sz += obj->bytes_size + 11;
// data tupe
sz += 2;
// file_size
sz += 11;
// block sizes
struct UnixFSBlockSizeNode* currNode = obj->block_size_head;
while(currNode != NULL) {
sz += 11;
currNode = currNode->next;
}
return sz;
}
/***
* Encode a UnixFS object into protobuf format
* @param incoming the incoming object
* @param outgoing where the bytes will be placed
* @param max_buffer_size the size of the outgoing buffer
* @param bytes_written how many bytes were written in the buffer
* @returns true(1) on success
*/
int ipfs_unixfs_protobuf_encode(struct UnixFS* incoming, unsigned char* outgoing, size_t max_buffer_size, size_t* bytes_written) {
size_t bytes_used = 0;
*bytes_written = 0;
int retVal = 0;
if (incoming != NULL) {
// data type (required)
retVal = protobuf_encode_varint(1, ipfs_unixfs_message_fields[0], incoming->data_type, outgoing, max_buffer_size - *bytes_written, &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
// bytes (optional)
if (incoming->bytes_size > 0) {
retVal = protobuf_encode_length_delimited(2, ipfs_unixfs_message_fields[1], (char*)incoming->bytes, incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
}
// file size (optional)
if (incoming->file_size > 0) {
retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
}
// block sizes
struct UnixFSBlockSizeNode* currNode = incoming->block_size_head;
while (currNode != NULL) {
retVal = protobuf_encode_varint(4, ipfs_unixfs_message_fields[3], currNode->block_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
currNode = currNode->next;
}
}
return 1;
}
/***
* Decodes a protobuf array of bytes into a UnixFS object
* @param incoming the array of bytes
* @param incoming_size the length of the array
* @param outgoing the UnixFS object
*/
int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, struct UnixFS** outgoing) {
// short cut for nulls
if (incoming_size == 0) {
*outgoing = NULL;
return 1;
}
size_t pos = 0;
int retVal = 0;
if (ipfs_unixfs_new(outgoing) == 0) {
return 0;
}
struct UnixFS* result = *outgoing;
while(pos < incoming_size) {
size_t bytes_read = 0;
int field_no;
enum WireType field_type;
if (protobuf_decode_field_and_type(&incoming[pos], incoming_size, &field_no, &field_type, &bytes_read) == 0) {
return 0;
}
pos += bytes_read;
switch(field_no) {
case (1): // data type (varint)
result->data_type = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
pos += bytes_read;
break;
case (2): // bytes (length delimited)
retVal = protobuf_decode_length_delimited(&incoming[pos], incoming_size - pos, (char**)&(result->bytes), &(result->bytes_size), &bytes_read);
if (retVal == 0)
return 0;
pos += bytes_read;
break;
case (3): // file size
result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
pos += bytes_read;
break;
case (4): { // block sizes (linked list from varint)
struct UnixFSBlockSizeNode* currNode = (struct UnixFSBlockSizeNode*)malloc(sizeof(struct UnixFSBlockSizeNode));
if (currNode == NULL)
return 0;
currNode->next = NULL;
currNode->block_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
pos += bytes_read;
if (result->block_size_head == NULL) {
result->block_size_head = currNode;
} else {
struct UnixFSBlockSizeNode* last = result->block_size_head;
while (last->next != NULL)
last = last->next;
last->next = currNode;
}
break;
}
}
}
return 1;
}