Intermediate save for hash matching

importer/exporter.c

@@ -96,7 +96,7 @@ int ipfs_exporter_to_console(const unsigned char* hash, const struct FSRepo* fs_
 
 	// process blocks
 	struct NodeLink* link = read_node->head_link;
-	printf("Links:[");
+	printf("{Links:[");
 	while (link != NULL) {
 		unsigned char b58[100];
 		ipfs_cid_hash_to_base58(link->hash, link->hash_size, b58, 100);
@@ -139,9 +139,13 @@ int ipfs_exporter_object_get(int argc, char** argv) {
 }
 
 int ipfs_exporter_cat_node(struct Node* node, const struct FSRepo* fs_repo) {
-	// process this block, then move on to the links
+	// process this node, then move on to the links
-	for(size_t i = 0LU; i < node->data_size; i++) {
+
-		printf("%c", node->data[i]);
+	// build the unixfs
+	struct UnixFS* unix_fs;
+	ipfs_unixfs_protobuf_decode(node->data, node->data_size, &unix_fs);
+	for(size_t i = 0LU; i < unix_fs->bytes_size; i++) {
+		printf("%c", unix_fs->bytes[i]);
 	}
 	// process links
 	struct NodeLink* current = node->head_link;

importer/importer.c

@@ -13,6 +13,58 @@
  * Imports OS files into the datastore
  */
 
+/***
+ * adds a blocksize to the UnixFS structure stored in the data
+ * element of a Node
+ * @param node the node to work with
+ * @param blocksize the blocksize to add
+ * @returns true(1) on success
+ */
+int ipfs_importer_add_filesize_to_data_section(struct Node* node, size_t bytes_read) {
+	// now add to the data section
+	struct UnixFS* data_section = NULL;
+	if (node->data == NULL) {
+		// nothing in data section yet, create new UnixFS
+		ipfs_unixfs_new(&data_section);
+		data_section->data_type = UNIXFS_FILE;
+	} else {
+		ipfs_unixfs_protobuf_decode(node->data, node->data_size, &data_section);
+	}
+	struct UnixFSBlockSizeNode bs;
+	bs.block_size = bytes_read;
+	ipfs_unixfs_add_blocksize(&bs, data_section);
+	data_section->file_size += bytes_read;
+	// put the new data back in the data section
+	size_t protobuf_size = ipfs_unixfs_protobuf_encode_size(data_section); //delay bytes_size entry
+	unsigned char protobuf[protobuf_size];
+	ipfs_unixfs_protobuf_encode(data_section, protobuf, protobuf_size, &protobuf_size);
+	ipfs_unixfs_free(data_section);
+	ipfs_node_set_data(node, protobuf, protobuf_size);
+	return 1;
+}
+
+int test_hash(unsigned char* protobuf, size_t protobuf_length) {
+	size_t hash_size = 32;
+	unsigned char* hash = (unsigned char*)malloc(hash_size);
+	if (hash == NULL) {
+		return 0;
+	}
+	if (libp2p_crypto_hashing_sha256(protobuf, protobuf_length, hash) == 0) {
+		return 0;
+	}
+
+	// turn it into base58
+	size_t buffer_len = 100;
+	unsigned char buffer[buffer_len];
+	int retVal = ipfs_cid_hash_to_base58(hash, hash_size, buffer, buffer_len);
+	if (retVal == 0) {
+		printf("Unable to generate hash\n");
+		return 0;
+	}
+	printf(" hash generated from %lu: %s\n", protobuf_length, buffer);
+	return 1;
+}
+
 /**
  * read the next chunk of bytes, create a node, and add a link to the node in the passed-in node
  * @param file the file handle
@@ -28,10 +80,15 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
 	struct Node* new_node = NULL;
 	struct NodeLink* new_link = NULL;
 
+	//JMJ
+	printf("Raw from the file");
+	test_hash(buffer, bytes_read);
+
 	// put the file bits into a new UnixFS file
 	if (ipfs_unixfs_new(&new_unixfs) == 0)
 		return 0;
 	new_unixfs->data_type = UNIXFS_FILE;
+	new_unixfs->file_size = bytes_read;
 	if (ipfs_unixfs_add_data(&buffer[0], bytes_read, new_unixfs) == 0) {
 		ipfs_unixfs_free(new_unixfs);
 		return 0;
@@ -48,6 +105,9 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
 		ipfs_unixfs_free(new_unixfs);
 		return 0;
 	}
+	//JMJ
+	printf("unixfs protobuf");
+	test_hash(protobuf, protobuf_size);
 	// we're done with the UnixFS object
 	ipfs_unixfs_free(new_unixfs);
 
@@ -76,6 +136,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
 			ipfs_node_free(new_node);
 			return 0;
 		}
+		ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read);
 		ipfs_node_free(new_node);
 	} else {
 		// if there are no existing links, put what we pulled from the file into parent_node
@@ -107,6 +168,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
 				ipfs_node_free(new_node);
 				return 0;
 			}
+			ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read);
 			ipfs_node_free(new_node);
 		}
 		// persist the main node

include/ipfs/unixfs/unixfs.h

@@ -43,7 +43,7 @@ struct UnixFS {
 	enum UnixFSDataType data_type;
 	size_t bytes_size; // the size of the bytes array
 	unsigned char* bytes; // an array of bytes
-	// size_t file_size; // the file size - I mimick this one
+	size_t file_size; // when saving files that have been chunked
 	struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes
 	unsigned char* hash; // not saved
 	size_t hash_length; // not saved
@@ -76,6 +76,8 @@ int ipfs_unixfs_free(struct UnixFS* obj);
  */
 int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs);
 
+int ipfs_unixfs_add_blocksize(const struct UnixFSBlockSizeNode* blocksize, struct UnixFS* unix_fs);
+
 /**
  * Protobuf functions
  */

merkledag/merkledag.c

@@ -6,6 +6,8 @@
 #include <string.h>
 
 #include "libp2p/crypto/sha256.h"
+#include "mh/multihash.h"
+#include "mh/hashes.h"
 #include "ipfs/merkledag/merkledag.h"
 #include "ipfs/unixfs/unixfs.h"
 

merkledag/node.c

@@ -6,9 +6,13 @@
 #include <stdio.h>
 #include <string.h>
 #include "inttypes.h"
-#include "ipfs/cid/cid.h"
 
+#include "mh/multihash.h"
+#include "mh/hashes.h"
+#include "ipfs/cid/cid.h"
 #include "ipfs/merkledag/node.h"
+#include "ipfs/unixfs/unixfs.h"
+
 
 // for protobuf Node (all fields optional)    data (optional bytes)      links (repeated node_link)
 enum WireType ipfs_node_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED };
@@ -116,14 +120,23 @@ int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* b
 	*bytes_written = 0;
 	// hash
 	if (link->hash_size > 0) {
-		retVal = protobuf_encode_length_delimited(1, ipfs_node_link_message_fields[0], (char*)link->hash, link->hash_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
+		size_t hash_length = mh_new_length(MH_H_SHA2_256, link->hash_size);
+		unsigned char hash[hash_length];
+		mh_new(hash, MH_H_SHA2_256, link->hash, link->hash_size);
+		retVal = protobuf_encode_length_delimited(1, ipfs_node_link_message_fields[0], (char*)hash, hash_length, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
+		if (retVal == 0) {
+			return 0;
+		}
+		*bytes_written += bytes_used;
+	}
+	// name is optional, but they encode it anyways
+	if (link->name != NULL && strlen(link->name) > 0) {
+		retVal = protobuf_encode_string(2, ipfs_node_link_message_fields[1], link->name, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
 		if (retVal == 0)
 			return 0;
 		*bytes_written += bytes_used;
-	}
+	} else {
-	// name
+		retVal = protobuf_encode_string(2, ipfs_node_link_message_fields[1], "", &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
-	if (link->name != NULL && strlen(link->name) > 0) {
-		retVal = protobuf_encode_string(2, ipfs_node_link_message_fields[1], link->name, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
 		if (retVal == 0)
 			return 0;
 		*bytes_written += bytes_used;
@@ -166,8 +179,15 @@ int ipfs_node_link_protobuf_decode(unsigned char* buffer, size_t buffer_length,
 		pos += bytes_read;
 		switch(field_no) {
 			case (1): { // hash
-				if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&link->hash, &link->hash_size, &bytes_read) == 0)
+				size_t hash_size = 0;
+				unsigned char* hash;
+				if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&hash, &hash_size, &bytes_read) == 0)
 					goto exit;
+				link->hash = (unsigned char*)malloc(hash_size - 2);
+				link->hash_size = hash_size - 2;
+				strncpy((char*)link->hash, (char*)hash, link->hash_size);
+				link->hash[hash_size-1] = 0;
+				free(hash);
 				pos += bytes_read;
 				break;
 			}
@@ -227,13 +247,6 @@ int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, si
 	size_t bytes_used = 0;
 	*bytes_written = 0;
 	int retVal = 0;
-	// data
-	if (node->data_size > 0) {
-		retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
-		if (retVal == 0)
-			return 0;
-		*bytes_written += bytes_used;
-	}
 	// links
 	struct NodeLink* current = node->head_link;
 	while (current != NULL) {
@@ -248,6 +261,13 @@ int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, si
 		*bytes_written += bytes_used;
 		current = current->next;
 	}
+	// data
+	if (node->data_size > 0) {
+		retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
+		if (retVal == 0)
+			return 0;
+		*bytes_written += bytes_used;
+	}
 
 	return 1;
 }
@@ -369,18 +389,21 @@ int ipfs_node_set_hash(struct Node* node, const unsigned char* hash, size_t hash
  * Sets pointers of encoded & cached to NULL /following go method
  * returns 1 on success 0 on failure
  */
-int ipfs_node_set_data(struct Node * N, unsigned char * Data, size_t data_size)
+int ipfs_node_set_data(struct Node* node, unsigned char * Data, size_t data_size)
 {
-	if(!N || !Data)
+	if(!node || !Data)
 	{
 		return 0;
 	}
-	N->data = malloc(sizeof(unsigned char) * data_size);
+	if (node->data != NULL) {
-	if (N->data == NULL)
+		free(node->data);
+	}
+	node->data = malloc(sizeof(unsigned char) * data_size);
+	if (node->data == NULL)
 		return 0;
 
-	memcpy(N->data, Data, data_size);
+	memcpy(node->data, Data, data_size);
-	N->data_size = data_size;
+	node->data_size = data_size;
 	return 1;
 }
 
@@ -527,17 +550,16 @@ int ipfs_node_remove_link_by_name(char * Name, struct Node * mynode)
 }
 
 /* ipfs_node_add_link
- * Adds a link to your nodse
+ * Adds a link to your node
- * @param mynode: &yournode
+ * @param node the node to add to
- * @param mylink: the CID you want to create a node from
+ * @param mylink: the link to add
- * @param linksz: sizeof(your cid here)
+ * @returns true(1) on success
- * Returns your node with the newly added link
  */
-int ipfs_node_add_link(struct Node* Nl, struct NodeLink * mylink)
+int ipfs_node_add_link(struct Node* node, struct NodeLink * mylink)
 {
-	if(Nl->head_link != NULL) {
+	if(node->head_link != NULL) {
 		// add to existing by finding last one
-		struct NodeLink* current_end = Nl->head_link;
+		struct NodeLink* current_end = node->head_link;
 		while(current_end->next != NULL) {
 			current_end = current_end->next;
 		}
@@ -546,7 +568,7 @@ int ipfs_node_add_link(struct Node* Nl, struct NodeLink * mylink)
 	}
 	else
 	{
-		Nl->head_link = mylink;
+		node->head_link = mylink;
 	}
 	return 1;
 }

unixfs/unixfs.c

@@ -48,6 +48,7 @@ int ipfs_unixfs_new(struct UnixFS** obj) {
 	(*obj)->block_size_head = NULL;
 	(*obj)->hash = NULL;
 	(*obj)->hash_length = 0;
+	(*obj)->file_size = 0;
 	return 1;
 }
 
@@ -98,6 +99,26 @@ int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS*
 
 }
 
+int ipfs_unixfs_add_blocksize(const struct UnixFSBlockSizeNode* blocksize, struct UnixFS* unix_fs) {
+	struct UnixFSBlockSizeNode* last = unix_fs->block_size_head;
+
+	if (last == NULL) {
+		// we're the first one
+		unix_fs->block_size_head = (struct UnixFSBlockSizeNode*)malloc(sizeof(struct UnixFSBlockSizeNode));
+		unix_fs->block_size_head->block_size = blocksize->block_size;
+	} else {
+		// find the last one
+		while (last->next != NULL) {
+			last = last->next;
+		}
+		last->next = (struct UnixFSBlockSizeNode*)malloc(sizeof(struct UnixFSBlockSizeNode));
+		last->next->block_size = blocksize->block_size;
+	}
+
+
+	return 1;
+}
+
 
 /**
  * Protobuf functions
@@ -154,8 +175,8 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou
 			*bytes_written += bytes_used;
 		}
 		// file size (optional)
-		if (incoming->data_type == UNIXFS_FILE && incoming->bytes_size > 0) {
+		if (incoming->file_size > 0) {
-			retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
+			retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
 			if (retVal == 0)
 				return 0;
 			*bytes_written += bytes_used;
@@ -164,6 +185,7 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou
 		struct UnixFSBlockSizeNode* currNode = incoming->block_size_head;
 		while (currNode != NULL) {
 			retVal = protobuf_encode_varint(4, ipfs_unixfs_message_fields[3], currNode->block_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
+			*bytes_written += bytes_used;
 			currNode = currNode->next;
 		}
 	}
@@ -211,7 +233,7 @@ int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, s
 				pos += bytes_read;
 				break;
 			case (3): // file size
-				result->bytes_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
+				result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
 				pos += bytes_read;
 				break;
 			case (4): { // block sizes (linked list from varint)