Intermidiate commit with big changes to storage formats

I am attempting to match the storage format of the reference
implementation, so as to generate the same hashes.

Makefile

@@ -16,6 +16,7 @@ all:
 	cd flatfs; make all;
 	cd datastore; make all;
 	cd thirdparty; make all;
+	cd unixfs; make all;
 	cd main; make all;
 	cd test; make all;
 	
@@ -33,6 +34,7 @@ clean:
 	cd flatfs; make clean;
 	cd datastore; make clean;
 	cd thirdparty; make clean;
+	cd unixfs; make clean;
 	cd main; make clean;
 	cd test; make clean;
 

blocks/blockstore.c

@@ -40,12 +40,12 @@ unsigned char* ipfs_blockstore_cid_to_base32(const struct Cid* cid) {
 	return buffer;
 }
 
-unsigned char* ipfs_blockstore_hash_to_base32(const unsigned char* hash, size_t hash_size) {
+unsigned char* ipfs_blockstore_hash_to_base32(const unsigned char* hash, size_t hash_length) {
-	size_t key_length = libp2p_crypto_encoding_base32_encode_size(hash_size);
+	size_t key_length = libp2p_crypto_encoding_base32_encode_size(hash_length);
 	unsigned char* buffer = (unsigned char*)malloc(key_length + 1);
 	if (buffer == NULL)
 		return NULL;
-	int retVal = ipfs_datastore_helper_ds_key_from_binary(hash, hash_size, &buffer[0], key_length, &key_length);
+	int retVal = ipfs_datastore_helper_ds_key_from_binary(hash, hash_length, &buffer[0], key_length, &key_length);
 	if (retVal == 0) {
 		free(buffer);
 		return NULL;
@@ -145,3 +145,135 @@ int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) {
 	free(filename);
 	return 1;
 }
+
+/***
+ * Put a struct UnixFS in the blockstore
+ * @param unix_fs the structure
+ * @param fs_repo the repo to place the strucure in
+ * @param bytes_written the number of bytes written to the blockstore
+ * @returns true(1) on success
+ */
+int ipfs_blockstore_put_unixfs(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written) {
+	// from blockstore.go line 118
+	int retVal = 0;
+
+	// Get Datastore key, which is a base32 key of the multihash,
+	unsigned char* key = ipfs_blockstore_hash_to_base32(unix_fs->hash, unix_fs->hash_length);
+	if (key == NULL) {
+		free(key);
+		return 0;
+	}
+
+	//TODO: put this in subdirectories
+
+	// turn the block into a binary array
+	size_t protobuf_len = ipfs_unixfs_protobuf_encode_size(unix_fs);
+	unsigned char protobuf[protobuf_len];
+	retVal = ipfs_unixfs_protobuf_encode(unix_fs, protobuf, protobuf_len, &protobuf_len);
+	if (retVal == 0) {
+		free(key);
+		return 0;
+	}
+
+	// now write byte array to file
+	char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
+	if (filename == NULL) {
+		free(key);
+		return 0;
+	}
+
+	FILE* file = fopen(filename, "wb");
+	*bytes_written = fwrite(protobuf, 1, protobuf_len, file);
+	fclose(file);
+	if (*bytes_written != protobuf_len) {
+		free(key);
+		free(filename);
+		return 0;
+	}
+
+	// send to Put with key (this is now done separately)
+	//fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
+
+	free(key);
+	free(filename);
+	return 1;
+}
+
+/***
+ * Find a UnixFS struct based on its hash
+ * @param hash the hash to look for
+ * @param hash_length the length of the hash
+ * @param unix_fs the struct to fill
+ * @param fs_repo where to look for the data
+ * @returns true(1) on success
+ */
+int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, struct UnixFS** block, const struct FSRepo* fs_repo) {
+	// get datastore key, which is a base32 key of the multihash
+	unsigned char* key = ipfs_blockstore_hash_to_base32(hash, hash_length);
+
+	char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
+
+	size_t file_size = os_utils_file_size(filename);
+	unsigned char buffer[file_size];
+
+	FILE* file = fopen(filename, "rb");
+	size_t bytes_read = fread(buffer, 1, file_size, file);
+	fclose(file);
+
+	int retVal = ipfs_unixfs_protobuf_decode(buffer, bytes_read, block);
+
+	free(key);
+	free(filename);
+
+	return retVal;
+}
+
+/***
+ * Put a struct Node in the blockstore
+ * @param node the structure
+ * @param fs_repo the repo to place the strucure in
+ * @param bytes_written the number of bytes written to the blockstore
+ * @returns true(1) on success
+ */
+int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written) {
+	// from blockstore.go line 118
+	int retVal = 0;
+
+	// Get Datastore key, which is a base32 key of the multihash,
+	unsigned char* key = ipfs_blockstore_hash_to_base32(node->hash, node->hash_size);
+	if (key == NULL) {
+		free(key);
+		return 0;
+	}
+
+	//TODO: put this in subdirectories
+
+	// turn the block into a binary array
+	size_t protobuf_len = ipfs_node_protobuf_encode_size(node);
+	unsigned char protobuf[protobuf_len];
+	retVal = ipfs_node_protobuf_encode(node, protobuf, protobuf_len, &protobuf_len);
+	if (retVal == 0) {
+		free(key);
+		return 0;
+	}
+
+	// now write byte array to file
+	char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
+	if (filename == NULL) {
+		free(key);
+		return 0;
+	}
+
+	FILE* file = fopen(filename, "wb");
+	*bytes_written = fwrite(protobuf, 1, protobuf_len, file);
+	fclose(file);
+	if (*bytes_written != protobuf_len) {
+		free(key);
+		free(filename);
+		return 0;
+	}
+
+	free(key);
+	free(filename);
+	return 1;
+}

importer/importer.c

@@ -23,27 +23,43 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
 	unsigned char buffer[MAX_DATA_SIZE];
 	size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file);
 
+	// put the file bits into a new UnixFS file
+	struct UnixFS* new_unixfs = NULL;
+	ipfs_unixfs_new(&new_unixfs);
+	new_unixfs->data_type = UNIXFS_FILE;
+	ipfs_unixfs_add_data(&buffer[0], bytes_read, new_unixfs);
+	// protobuf the UnixFS
+	size_t protobuf_size = ipfs_unixfs_protobuf_encode_size(new_unixfs);
+	unsigned char protobuf[protobuf_size];
+	size_t bytes_written = 0;
+	ipfs_unixfs_protobuf_encode(new_unixfs, protobuf, protobuf_size, &bytes_written);
+	// we're done with the object
+	ipfs_unixfs_free(new_unixfs);
 	// create a new node
 	struct Node* new_node = NULL;
-	ipfs_node_new_from_data(buffer, bytes_read, &new_node);
+	ipfs_node_new_from_data(protobuf, bytes_written, &new_node);
+	ipfs_node_set_hash(new_node, new_unixfs->hash, new_unixfs->hash_length);
 	// persist
-	ipfs_merkledag_add(new_node, fs_repo);
+	size_t size_of_node = 0;
+	ipfs_merkledag_add(new_node, fs_repo, &size_of_node);
 	// put link in parent node
 	struct NodeLink* new_link = NULL;
 	ipfs_node_link_create("", new_node->hash, new_node->hash_size, &new_link);
-	new_link->t_size = new_node->data_size;
+	new_link->t_size = size_of_node;
 	*total_size += new_link->t_size;
 	ipfs_node_add_link(parent_node, new_link);
 	ipfs_node_free(new_node);
-	// save the parent_node if it is time...
 	if (bytes_read != MAX_DATA_SIZE) {
-		// build UnixFS for file
+		// We have read everything, now save the parent_node,
+		// which is a sort of "directory" entry
+		/*
+		// build UnixFS for the parent
 		struct UnixFS* unix_fs;
 		if (ipfs_unixfs_new(&unix_fs) == 0) {
 			return 0;
 		}
 		unix_fs->data_type = UNIXFS_FILE;
-		unix_fs->file_size = *total_size;
+		unix_fs->bytes_size = *total_size;
 		// now encode unixfs and put in parent_node->data
 		size_t temp_size = ipfs_unixfs_protobuf_encode_size(unix_fs);
 		unsigned char temp[temp_size];
@@ -59,8 +75,10 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
 			return 0;
 		}
 		memcpy(parent_node->data, temp, bytes_written);
+		ipfs_unixfs_free(unix_fs);
+		*/
 		// persist the main node
-		ipfs_merkledag_add(parent_node, fs_repo);
+		ipfs_merkledag_add(parent_node, fs_repo, &bytes_written);
 	}
 	return bytes_read;
 }

include/ipfs/blocks/blockstore.h

@@ -37,5 +37,28 @@ int ipfs_blockstore_get(const unsigned char* hash, size_t hash_length, struct Bl
  */
 int ipfs_blockstore_put(struct Block* block, const struct FSRepo* fs_repo);
 
+/***
+ * Put a struct UnixFS in the blockstore
+ * @param unix_fs the structure
+ * @param fs_repo the repo to place the strucure in
+ * @param bytes_written the number of bytes written to the blockstore
+ * @returns true(1) on success
+ */
+int ipfs_blockstore_put_unixfs(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written);
+
+/***
+ * Find a UnixFS struct based on its hash
+ * @param hash the hash to look for
+ * @param hash_length the length of the hash
+ * @param unix_fs the struct to fill
+ * @param fs_repo where to look for the data
+ * @returns true(1) on success
+ */
+int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, struct UnixFS** block, const struct FSRepo* fs_repo);
+
+/**
+ * Put a struct Node in the blockstore
+ */
+int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written);
 
 #endif

include/ipfs/merkledag/merkledag.h

@@ -10,10 +10,11 @@
 /***
  * Adds a node to the dagService and blockService
  * @param node the node to add
- * @param cid the resultant cid that was added
+ * @param fs_repo the repo to add to
+ * @param bytes_written the number of bytes written
  * @returns true(1) on success
  */
-int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo);
+int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_written);
 
 /***
  * Retrieves a node from the datastore based on the cid

include/ipfs/merkledag/node.h

@@ -74,7 +74,7 @@ int ipfs_node_link_free(struct NodeLink * node_link);
  * @param link the link to examine
  * @returns the maximum size that should be needed
  */
-size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link);
+size_t ipfs_node_link_protobuf_encode_size(const struct NodeLink* link);
 
 /***
  * Encode a NodeLink into protobuf format
@@ -84,7 +84,7 @@ size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link);
  * @pram bytes_written the amount of the buffer used
  * @returns true(1) on success
  */
-int ipfs_node_link_protobuf_encode(struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
+int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
 
 /****
  * Decode from a byte array into a NodeLink
@@ -102,7 +102,7 @@ int ipfs_node_link_protobuf_decode(unsigned char* buffer, size_t buffer_length,
  * @param node the node to examine
  * @returns the max size of an encoded stream of bytes, if it were encoded
  */
-size_t ipfs_node_protobuf_encode_size(struct Node* node);
+size_t ipfs_node_protobuf_encode_size(const struct Node* node);
 
 /***
  * Encode a node into a protobuf byte stream
@@ -112,7 +112,7 @@ size_t ipfs_node_protobuf_encode_size(struct Node* node);
  * @param bytes_written how much of buffer was used
  * @returns true(1) on success
  */
-int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
+int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
 
 /***
  * Decode a stream of bytes into a Node structure

include/ipfs/repo/fsrepo/fs_repo.h

@@ -5,8 +5,10 @@
 #define fs_repo_h
 
 #include <stdio.h>
-#include "ipfs/repo/config/config.h"
 
+#include "ipfs/repo/config/config.h"
+#include "ipfs/unixfs/unixfs.h"
+#include "ipfs/merkledag/node.h"
 
 /**
  * a structure to hold the repo info
@@ -73,4 +75,23 @@ int ipfs_repo_fsrepo_init(struct FSRepo* config);
 int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo);
 int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo);
 
+/***
+ * Write a unixfs to the datastore and blockstore
+ * @param unix_fs the struct to write
+ * @param fs_repo the repo to write to
+ * @returns true(1) on success
+ */
+int ipfs_repo_fsrepo_unixfs_write(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written);
+int ipfs_repo_fsrepo_unixfs_read(const unsigned char* hash, size_t hash_length, struct UnixFS** unix_fs, const struct FSRepo* fs_repo);
+
+/***
+ * Write a struct Node to the datastore and blockstore
+ * @param node the struct to write
+ * @param fs_repo the repo to write to
+ * @returns true(1) on success
+ */
+int ipfs_repo_fsrepo_node_write(const struct Node* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written);
+int ipfs_repo_fsrepo_node_read(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo);
+
+
 #endif /* fs_repo_h */

include/ipfs/unixfs/format.h

@@ -1,4 +1,4 @@
-enum UnixFSFormatType { RAW, FILE, DIRECTORY, METADATA, SYMLINK };
+enum UnixFSFormatType { RAW, UNIXFS_FILE, DIRECTORY, METADATA, SYMLINK };
 
 struct UnixFSData {
 	enum UnixFSFormatType type;

include/ipfs/unixfs/unixfs.h

@@ -43,8 +43,10 @@ struct UnixFS {
 	enum UnixFSDataType data_type;
 	size_t bytes_size; // the size of the bytes array
 	unsigned char* bytes; // an array of bytes
-	size_t file_size; // the file size
+	// size_t file_size; // the file size - I mimick this one
 	struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes
+	unsigned char* hash; // not saved
+	size_t hash_length; // not saved
 };
 
 struct UnixFSMetaData {
@@ -65,6 +67,15 @@ int ipfs_unixfs_new(struct UnixFS** obj);
  */
 int ipfs_unixfs_free(struct UnixFS* obj);
 
+/***
+ * Write data to data section of a UnixFS stuct. NOTE: this also calculates a sha256 hash
+ * @param data the data to write
+ * @param data_length the length of the data
+ * @param unix_fs the struct to add to
+ * @returns true(1) on success
+ */
+int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs);
+
 /**
  * Protobuf functions
  */
@@ -74,7 +85,7 @@ int ipfs_unixfs_free(struct UnixFS* obj);
  * @param obj what will be encoded
  * @returns the size of the buffer necessary to encode the object
  */
-size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj);
+size_t ipfs_unixfs_protobuf_encode_size(const struct UnixFS* obj);
 
 /***
  * Encode a UnixFS object into protobuf format

main/Makefile

@@ -9,14 +9,14 @@ OBJS = main.o \
 	../commands/argument.o ../commands/command_option.o ../commands/command.o ../commands/cli/parse.o \
 	../core/builder.o \
 	../datastore/ds_helper.o \
+	../dnslink/dnslink.o \
 	../flatfs/flatfs.o \
 	../importer/importer.o \
 	../importer/exporter.o \
-	../dnslink/dnslink.o \
 	../path/path.o \
-	../namesys/isdomain.o \
 	../merkledag/merkledag.o ../merkledag/node.o \
 	../multibase/multibase.o \
+	../namesys/isdomain.o \
 	../os/utils.o \
 	../repo/init.o \
 	../repo/fsrepo/fs_repo.o ../repo/fsrepo/jsmn.o ../repo/fsrepo/lmdb_datastore.o \

merkledag/merkledag.c

@@ -1,41 +1,46 @@
 /**
  * A basic storage building block of the IPFS system
  */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
+#include "libp2p/crypto/sha256.h"
 #include "ipfs/merkledag/merkledag.h"
+#include "ipfs/unixfs/unixfs.h"
 
 /***
  * Adds a node to the dagService and blockService
  * @param node the node to add
- * @param cid the resultant cid that was added
+ * @param fs_repo the repo to add to
+ * @param bytes_written the number of bytes written
  * @returns true(1) on success
  */
-int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
+int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_written) {
 	// taken from merkledag.go line 59
+	int retVal = 0;
 
-	struct Block* block = NULL;
+	// compute the hash
+	size_t protobuf_size = ipfs_node_protobuf_encode_size(node);
+	unsigned char protobuf[protobuf_size];
+	size_t bytes_encoded;
+	retVal = ipfs_node_protobuf_encode(node, protobuf, protobuf_size, &bytes_encoded);
 
-	// protobuf the node
+	node->hash_size = 32;
-	size_t protobuf_len = ipfs_node_protobuf_encode_size(node);
+	node->hash = (unsigned char*)malloc(node->hash_size);
-	size_t bytes_written = 0;
+	if (node->hash == NULL) {
-	unsigned char protobuf[protobuf_len];
+		return 0;
-	ipfs_node_protobuf_encode(node, protobuf, protobuf_len, &bytes_written);
+	}
-
+	if (libp2p_crypto_hashing_sha256(protobuf, bytes_encoded, &node->hash[0]) == 0) {
-	// turn the node into a block
+		free(node->hash);
-	ipfs_blocks_block_new(&block);
-	ipfs_blocks_block_add_data(protobuf, bytes_written, block);
-
-	// write to block store & datastore
-	int retVal = ipfs_repo_fsrepo_block_write(block, fs_repo);
-	if (retVal == 0) {
-		ipfs_blocks_block_free(block);
 		return 0;
 	}
 
-	ipfs_node_set_hash(node, block->cid->hash, block->cid->hash_length);
+	// write to block store & datastore
-
+	retVal = ipfs_repo_fsrepo_node_write(node, fs_repo, bytes_written);
-	if (block != NULL)
+	if (retVal == 0) {
-		ipfs_blocks_block_free(block);
+		return 0;
+	}
 
 	// TODO: call HasBlock (unsure why as yet)
 	return 1;
@@ -51,7 +56,8 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
  */
 int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node** node, const struct FSRepo* fs_repo) {
 	int retVal = 1;
-	struct Block* block;
+	//struct Block* block;
+	struct UnixFS* unix_fs;
 	size_t key_length = 100;
 	unsigned char key[key_length];
 
@@ -61,27 +67,27 @@ int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node*
 	if (retVal == 0)
 		return 0;
 
-	// we have the record from the db. Go get the block from the blockstore
+	// we have the record from the db. Go get the UnixFS from the blockstore
-	retVal = ipfs_repo_fsrepo_block_read(hash, hash_size, &block, fs_repo);
+	retVal = ipfs_repo_fsrepo_unixfs_read(hash, hash_size, &unix_fs, fs_repo);
 	if (retVal == 0) {
 		return 0;
 	}
 
 	// now convert the block into a node
-	if (ipfs_node_protobuf_decode(block->data, block->data_length, node) == 0) {
+	if (ipfs_node_protobuf_decode(unix_fs->bytes, unix_fs->bytes_size, node) == 0) {
-		ipfs_blocks_block_free(block);
+		ipfs_unixfs_free(unix_fs);
 		return 0;
 	}
 
 	// set the cid on the node
 	if (ipfs_node_set_hash(*node, hash, hash_size) == 0) {
-		ipfs_blocks_block_free(block);
+		ipfs_unixfs_free(unix_fs);
 		ipfs_node_free(*node);
 		return 0;
 	}
 
 	// free resources
-	ipfs_blocks_block_free(block);
+	ipfs_unixfs_free(unix_fs);
 
 	return 1;
 }

merkledag/node.c

@@ -10,7 +10,7 @@
 
 #include "ipfs/merkledag/node.h"
 
-// for protobuf Node (all fields optional)  links (repeated node_link)   data (optional bytes)
+// for protobuf Node (all fields optional)    data (optional bytes)      links (repeated node_link)
 enum WireType ipfs_node_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED };
 // for protobuf NodeLink (all fields optional)       hash                            name                     tsize
 enum WireType ipfs_node_link_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED, WIRETYPE_VARINT };
@@ -84,7 +84,7 @@ int ipfs_node_link_free(struct NodeLink * node_link)
  * @param link the link to examine
  * @returns the maximum length of the encoded NodeLink
  */
-size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link) {
+size_t ipfs_node_link_protobuf_encode_size(const struct NodeLink* link) {
 	if (link == NULL)
 		return 0;
 	// hash, name, tsize
@@ -109,7 +109,7 @@ size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link) {
  * @param bytes_written the number of bytes written to buffer
  * @returns true(1) on success
  */
-int ipfs_node_link_protobuf_encode(struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
+int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
 	// 3 fields, hash (length delimited), name (length delimited), tsize (varint)
 	size_t bytes_used = 0;
 	int retVal = 0;
@@ -200,7 +200,7 @@ exit:
 /***
  * return an approximate size of the encoded node
  */
-size_t ipfs_node_protobuf_encode_size(struct Node* node) {
+size_t ipfs_node_protobuf_encode_size(const struct Node* node) {
 	size_t size = 0;
 	// links
 	struct NodeLink* current = node->head_link;
@@ -223,10 +223,17 @@ size_t ipfs_node_protobuf_encode_size(struct Node* node) {
  * @param bytes_written how much of buffer was used
  * @returns true(1) on success
  */
-int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
+int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
 	size_t bytes_used = 0;
 	*bytes_written = 0;
 	int retVal = 0;
+	// data
+	if (node->data_size > 0) {
+		retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
+		if (retVal == 0)
+			return 0;
+		*bytes_written += bytes_used;
+	}
 	// links
 	struct NodeLink* current = node->head_link;
 	while (current != NULL) {
@@ -235,19 +242,12 @@ int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t m
 		retVal = ipfs_node_link_protobuf_encode(current, temp, temp_size, &bytes_used);
 		if (retVal == 0)
 			return 0;
-		retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)temp, bytes_used, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
+		retVal = protobuf_encode_length_delimited(2, ipfs_node_message_fields[1], (char*)temp, bytes_used, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
 		if (retVal == 0)
 			return 0;
 		*bytes_written += bytes_used;
 		current = current->next;
 	}
-	// data
-	if (node->data_size > 0) {
-		retVal = protobuf_encode_length_delimited(2, ipfs_node_message_fields[1], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
-		if (retVal == 0)
-			return 0;
-		*bytes_written += bytes_used;
-	}
 
 	return 1;
 }
@@ -261,8 +261,8 @@ int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t m
  */
 int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struct Node** node) {
 	/*
-	 * Field 0: link
 	 * Field 1: data
+	 * Field 2: link
 	 */
 	size_t pos = 0;
 	int retVal = 0;
@@ -282,7 +282,13 @@ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struc
 		}
 		pos += bytes_read;
 		switch(field_no) {
-			case (1): {// links
+			case (1): { // data
+				if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*node)->data), &((*node)->data_size), &bytes_read) == 0)
+					goto exit;
+				pos += bytes_read;
+				break;
+			}
+			case (2): {// links
 				if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0)
 					goto exit;
 				pos += bytes_read;
@@ -293,12 +299,6 @@ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struc
 				ipfs_node_add_link(*node, temp_link);
 				break;
 			}
-			case (2): { // data
-				if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*node)->data), &((*node)->data_size), &bytes_read) == 0)
-					goto exit;
-				pos += bytes_read;
-				break;
-			}
 		}
 	}
 

repo/fsrepo/fs_repo.c

@@ -554,6 +554,69 @@ int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_re
 	return 1;
 }
 
+/***
+ * Write a unixfs to the datastore and blockstore
+ * @param unix_fs the struct to write
+ * @param fs_repo the repo to write to
+ * @param bytes_written number of bytes written to the repo
+ * @returns true(1) on success
+ */
+int ipfs_repo_fsrepo_unixfs_write(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written) {
+	/**
+	 * What is put in the blockstore is the block.
+	 * What is put in the datastore is the multihash (the Cid) as the key,
+	 * and the base32 encoded multihash as the value.
+	 */
+	int retVal = 1;
+	retVal = ipfs_blockstore_put_unixfs(unix_fs, fs_repo, bytes_written);
+	if (retVal == 0)
+		return 0;
+	// take the hash, base32 it, and send both to the datastore
+	size_t fs_key_length = 100;
+	unsigned char fs_key[fs_key_length];
+	retVal = ipfs_datastore_helper_ds_key_from_binary(unix_fs->hash, unix_fs->hash_length, fs_key, fs_key_length, &fs_key_length);
+	if (retVal == 0)
+		return 0;
+	retVal = fs_repo->config->datastore->datastore_put(unix_fs->hash, unix_fs->hash_length, fs_key, fs_key_length, fs_repo->config->datastore);
+	if (retVal == 0)
+		return 0;
+	return 1;
+}
+
+/***
+ * Write a unixfs to the datastore and blockstore
+ * @param unix_fs the struct to write
+ * @param fs_repo the repo to write to
+ * @param bytes_written number of bytes written to the repo
+ * @returns true(1) on success
+ */
+int ipfs_repo_fsrepo_node_write(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written) {
+	/**
+	 * What is put in the blockstore is the node.
+	 * What is put in the datastore is the multihash as the key,
+	 * and the base32 encoded multihash as the value.
+	 */
+	int retVal = 1;
+	retVal = ipfs_blockstore_put_node(node, fs_repo, bytes_written);
+	if (retVal == 0)
+		return 0;
+	// take the hash, base32 it, and send both to the datastore
+	size_t fs_key_length = 100;
+	unsigned char fs_key[fs_key_length];
+	retVal = ipfs_datastore_helper_ds_key_from_binary(node->hash, node->hash_size, fs_key, fs_key_length, &fs_key_length);
+	if (retVal == 0)
+		return 0;
+	retVal = fs_repo->config->datastore->datastore_put(node->hash, node->hash_size, fs_key, fs_key_length, fs_repo->config->datastore);
+	if (retVal == 0)
+		return 0;
+	return 1;
+}
+
+int ipfs_repo_fsrepo_node_read(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo) {
+	 return 0;
+}
+
+
 int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo) {
 	int retVal = 0;
 
@@ -569,3 +632,18 @@ int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, s
 	return retVal;
 }
 
+int ipfs_repo_fsrepo_unixfs_read(const unsigned char* hash, size_t hash_length, struct UnixFS** unix_fs, const struct FSRepo* fs_repo) {
+	int retVal = 0;
+
+	// get the base32 hash from the database
+	// We do this only to see if it is in the database
+	size_t fs_key_length = 100;
+	unsigned char fs_key[fs_key_length];
+	retVal = fs_repo->config->datastore->datastore_get((const char*)hash, hash_length, fs_key, fs_key_length, &fs_key_length, fs_repo->config->datastore);
+	if (retVal == 0) // maybe it doesn't exist?
+		return 0;
+	// now get the block from the blockstore
+	retVal = ipfs_blockstore_get_unixfs(hash, hash_length, unix_fs, fs_repo);
+	return retVal;
+}
+

test/merkledag/test_merkledag.h

@@ -42,8 +42,8 @@ int test_merkledag_get_data() {
 	// create a node
 	struct Node* node1;
 	retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1);
-
+	size_t bytes_written = 0;
-	retVal = ipfs_merkledag_add(node1, fs_repo);
+	retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
 	if (retVal == 0) {
 		ipfs_node_free(node1);
 		ipfs_repo_fsrepo_free(fs_repo);
@@ -105,7 +105,8 @@ int test_merkledag_add_data() {
 	struct Node* node1;
 	retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1);
 
-	retVal = ipfs_merkledag_add(node1, fs_repo);
+	size_t bytes_written = 0;
+	retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
 	if (retVal == 0) {
 		ipfs_node_free(node1);
 		return 0;
@@ -124,7 +125,7 @@ int test_merkledag_add_data() {
 	// adding the same binary again should do nothing (the hash should be the same)
 	struct Node* node2;
 	retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node2);
-	retVal = ipfs_merkledag_add(node2, fs_repo);
+	retVal = ipfs_merkledag_add(node2, fs_repo, &bytes_written);
 	if (retVal == 0) {
 		ipfs_node_free(node1);
 		ipfs_node_free(node2);
@@ -160,7 +161,7 @@ int test_merkledag_add_data() {
 	struct Node* node3;
 	retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node3);
 
-	retVal = ipfs_merkledag_add(node3, fs_repo);
+	retVal = ipfs_merkledag_add(node3, fs_repo, &bytes_written);
 	if (retVal == 0) {
 		ipfs_node_free(node1);
 		ipfs_node_free(node2);
@@ -207,7 +208,8 @@ int test_merkledag_add_node() {
 		return 0;
 	}
 
-	retVal = ipfs_merkledag_add(node1, fs_repo);
+	size_t bytes_written = 0;
+	retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
 	if (retVal == 0) {
 		ipfs_repo_fsrepo_free(fs_repo);
 		ipfs_node_free(node1);
@@ -250,7 +252,8 @@ int test_merkledag_add_node_with_links() {
 		return 0;
 	}
 
-	retVal = ipfs_merkledag_add(node1, fs_repo);
+	size_t bytes_written = 0;
+	retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
 	if (retVal == 0) {
 		ipfs_repo_fsrepo_free(fs_repo);
 		ipfs_node_free(node1);

test/storage/test_unixfs.h

@@ -65,7 +65,6 @@ int test_unixfs_encode_smallfile() {
 	memcpy(unixfs->bytes, bytes, 35);
 	unixfs->bytes_size = 35;
 	unixfs->data_type = UNIXFS_FILE;
-	unixfs->file_size = 35;
 
 	size_t protobuf_size = 43;
 	unsigned char protobuf[protobuf_size];

unixfs/Makefile

@@ -0,0 +1,18 @@
+CC = gcc
+CFLAGS = -O0 -I../include -I../../c-libp2p/include -I../../c-multihash/include -I../../c-multiaddr/include -I../../c-protobuf -Wall
+
+ifdef DEBUG
+CFLAGS += -g3
+endif
+
+LFLAGS = 
+DEPS = 
+OBJS = unixfs.o
+
+%.o: %.c $(DEPS)
+	$(CC) -c -o $@ $< $(CFLAGS)
+
+all: $(OBJS)
+
+clean:
+	rm -f *.o

unixfs/format.c.old

@@ -1,7 +1,10 @@
 #include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
 
 #include "ipfs/unixfs/format.h"
 
+
 /**
  * Copy data into a new struct
  * @param type the type of data
@@ -12,7 +15,7 @@
  */
 int ipfs_unixfs_format_new(enum UnixFSFormatType type, const unsigned char* data,
 		size_t data_length, struct UnixFSData** result) {
-	struct UnixFSData* new_struct = malloc(sizeof(struct UnixFSData));
+	struct UnixFSData* new_struct = (struct UnixFSData*)malloc(sizeof(struct UnixFSData));
 	if (new_struct == NULL)
 		return 0;
 

unixfs/unixfs.c

@@ -4,7 +4,9 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
+#include "libp2p/crypto/sha256.h"
 #include "ipfs/unixfs/unixfs.h"
 #include "protobuf.h"
 #include "varint.h"
@@ -43,8 +45,9 @@ int ipfs_unixfs_new(struct UnixFS** obj) {
 	(*obj)->bytes = 0;
 	(*obj)->bytes_size = 0;
 	(*obj)->data_type = UNIXFS_RAW;
-	(*obj)->file_size = 0;
 	(*obj)->block_size_head = NULL;
+	(*obj)->hash = NULL;
+	(*obj)->hash_length = 0;
 	return 1;
 }
 
@@ -56,6 +59,40 @@ int ipfs_unixfs_free(struct UnixFS* obj) {
 	return 1;
 }
 
+/***
+ * Write data to data section of a UnixFS stuct. NOTE: this also calculates a sha256 hash
+ * @param data the data to write
+ * @param data_length the length of the data
+ * @param unix_fs the struct to add to
+ * @returns true(1) on success
+ */
+int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs) {
+
+	unix_fs->bytes_size = data_length;
+	unix_fs->bytes = malloc(sizeof(unsigned char) * data_length);
+	if ( unix_fs->bytes == NULL) {
+		return 0;
+	}
+	memcpy( unix_fs->bytes, data, data_length);
+
+	// now compute the hash
+	unix_fs->hash_length = 32;
+	unix_fs->hash = (unsigned char*)malloc(unix_fs->hash_length);
+	if (unix_fs->hash == NULL) {
+		free(unix_fs->bytes);
+		return 0;
+	}
+	if (libp2p_crypto_hashing_sha256(data, data_length, &unix_fs->hash[0]) == 0) {
+		free(unix_fs->bytes);
+		free(unix_fs->hash);
+		return 0;
+	}
+
+	return 1;
+
+}
+
+
 /**
  * Protobuf functions
  */
@@ -68,7 +105,7 @@ enum WireType ipfs_unixfs_message_fields[] = { WIRETYPE_VARINT, WIRETYPE_LENGTH_
  * @param obj what will be encoded
  * @returns the size of the buffer necessary to encode the object
  */
-size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj) {
+size_t ipfs_unixfs_protobuf_encode_size(const struct UnixFS* obj) {
 	size_t sz = 0;
 	// bytes
 	sz += obj->bytes_size + 11;
@@ -111,8 +148,8 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou
 			*bytes_written += bytes_used;
 		}
 		// file size (optional)
-		if (incoming->file_size > 0) {
+		if (incoming->data_type == UNIXFS_FILE && incoming->bytes_size > 0) {
-			retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
+			retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
 			if (retVal == 0)
 				return 0;
 			*bytes_written += bytes_used;
@@ -168,7 +205,7 @@ int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, s
 				pos += bytes_read;
 				break;
 			case (3): // file size
-				result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
+				result->bytes_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
 				pos += bytes_read;
 				break;
 			case (4): { // block sizes (linked list from varint)