From 1dcbf8962e14d490ee331668966b3dff2bc54754 Mon Sep 17 00:00:00 2001 From: jmjatlanta Date: Fri, 23 Dec 2016 17:21:04 -0500 Subject: [PATCH] Intermediate save for hash matching --- importer/exporter.c | 12 ++++-- importer/importer.c | 62 ++++++++++++++++++++++++++++ include/ipfs/unixfs/unixfs.h | 4 +- merkledag/merkledag.c | 2 + merkledag/node.c | 80 +++++++++++++++++++++++------------- unixfs/unixfs.c | 28 +++++++++++-- 6 files changed, 151 insertions(+), 37 deletions(-) diff --git a/importer/exporter.c b/importer/exporter.c index 00fc683..77a1a6e 100644 --- a/importer/exporter.c +++ b/importer/exporter.c @@ -96,7 +96,7 @@ int ipfs_exporter_to_console(const unsigned char* hash, const struct FSRepo* fs_ // process blocks struct NodeLink* link = read_node->head_link; - printf("Links:["); + printf("{Links:["); while (link != NULL) { unsigned char b58[100]; ipfs_cid_hash_to_base58(link->hash, link->hash_size, b58, 100); @@ -139,9 +139,13 @@ int ipfs_exporter_object_get(int argc, char** argv) { } int ipfs_exporter_cat_node(struct Node* node, const struct FSRepo* fs_repo) { - // process this block, then move on to the links - for(size_t i = 0LU; i < node->data_size; i++) { - printf("%c", node->data[i]); + // process this node, then move on to the links + + // build the unixfs + struct UnixFS* unix_fs; + ipfs_unixfs_protobuf_decode(node->data, node->data_size, &unix_fs); + for(size_t i = 0LU; i < unix_fs->bytes_size; i++) { + printf("%c", unix_fs->bytes[i]); } // process links struct NodeLink* current = node->head_link; diff --git a/importer/importer.c b/importer/importer.c index 04cc936..a63778e 100644 --- a/importer/importer.c +++ b/importer/importer.c @@ -13,6 +13,58 @@ * Imports OS files into the datastore */ +/*** + * adds a blocksize to the UnixFS structure stored in the data + * element of a Node + * @param node the node to work with + * @param blocksize the blocksize to add + * @returns true(1) on success + */ +int ipfs_importer_add_filesize_to_data_section(struct Node* node, size_t bytes_read) { + // now add to the data section + struct UnixFS* data_section = NULL; + if (node->data == NULL) { + // nothing in data section yet, create new UnixFS + ipfs_unixfs_new(&data_section); + data_section->data_type = UNIXFS_FILE; + } else { + ipfs_unixfs_protobuf_decode(node->data, node->data_size, &data_section); + } + struct UnixFSBlockSizeNode bs; + bs.block_size = bytes_read; + ipfs_unixfs_add_blocksize(&bs, data_section); + data_section->file_size += bytes_read; + // put the new data back in the data section + size_t protobuf_size = ipfs_unixfs_protobuf_encode_size(data_section); //delay bytes_size entry + unsigned char protobuf[protobuf_size]; + ipfs_unixfs_protobuf_encode(data_section, protobuf, protobuf_size, &protobuf_size); + ipfs_unixfs_free(data_section); + ipfs_node_set_data(node, protobuf, protobuf_size); + return 1; +} + +int test_hash(unsigned char* protobuf, size_t protobuf_length) { + size_t hash_size = 32; + unsigned char* hash = (unsigned char*)malloc(hash_size); + if (hash == NULL) { + return 0; + } + if (libp2p_crypto_hashing_sha256(protobuf, protobuf_length, hash) == 0) { + return 0; + } + + // turn it into base58 + size_t buffer_len = 100; + unsigned char buffer[buffer_len]; + int retVal = ipfs_cid_hash_to_base58(hash, hash_size, buffer, buffer_len); + if (retVal == 0) { + printf("Unable to generate hash\n"); + return 0; + } + printf(" hash generated from %lu: %s\n", protobuf_length, buffer); + return 1; +} + /** * read the next chunk of bytes, create a node, and add a link to the node in the passed-in node * @param file the file handle @@ -28,10 +80,15 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs struct Node* new_node = NULL; struct NodeLink* new_link = NULL; + //JMJ + printf("Raw from the file"); + test_hash(buffer, bytes_read); + // put the file bits into a new UnixFS file if (ipfs_unixfs_new(&new_unixfs) == 0) return 0; new_unixfs->data_type = UNIXFS_FILE; + new_unixfs->file_size = bytes_read; if (ipfs_unixfs_add_data(&buffer[0], bytes_read, new_unixfs) == 0) { ipfs_unixfs_free(new_unixfs); return 0; @@ -48,6 +105,9 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs ipfs_unixfs_free(new_unixfs); return 0; } + //JMJ + printf("unixfs protobuf"); + test_hash(protobuf, protobuf_size); // we're done with the UnixFS object ipfs_unixfs_free(new_unixfs); @@ -76,6 +136,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs ipfs_node_free(new_node); return 0; } + ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read); ipfs_node_free(new_node); } else { // if there are no existing links, put what we pulled from the file into parent_node @@ -107,6 +168,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs ipfs_node_free(new_node); return 0; } + ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read); ipfs_node_free(new_node); } // persist the main node diff --git a/include/ipfs/unixfs/unixfs.h b/include/ipfs/unixfs/unixfs.h index e08b2f5..668c010 100644 --- a/include/ipfs/unixfs/unixfs.h +++ b/include/ipfs/unixfs/unixfs.h @@ -43,7 +43,7 @@ struct UnixFS { enum UnixFSDataType data_type; size_t bytes_size; // the size of the bytes array unsigned char* bytes; // an array of bytes - // size_t file_size; // the file size - I mimick this one + size_t file_size; // when saving files that have been chunked struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes unsigned char* hash; // not saved size_t hash_length; // not saved @@ -76,6 +76,8 @@ int ipfs_unixfs_free(struct UnixFS* obj); */ int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs); +int ipfs_unixfs_add_blocksize(const struct UnixFSBlockSizeNode* blocksize, struct UnixFS* unix_fs); + /** * Protobuf functions */ diff --git a/merkledag/merkledag.c b/merkledag/merkledag.c index aef5d06..09b353e 100644 --- a/merkledag/merkledag.c +++ b/merkledag/merkledag.c @@ -6,6 +6,8 @@ #include #include "libp2p/crypto/sha256.h" +#include "mh/multihash.h" +#include "mh/hashes.h" #include "ipfs/merkledag/merkledag.h" #include "ipfs/unixfs/unixfs.h" diff --git a/merkledag/node.c b/merkledag/node.c index 2434b3d..ebff28a 100644 --- a/merkledag/node.c +++ b/merkledag/node.c @@ -6,9 +6,13 @@ #include #include #include "inttypes.h" -#include "ipfs/cid/cid.h" +#include "mh/multihash.h" +#include "mh/hashes.h" +#include "ipfs/cid/cid.h" #include "ipfs/merkledag/node.h" +#include "ipfs/unixfs/unixfs.h" + // for protobuf Node (all fields optional) data (optional bytes) links (repeated node_link) enum WireType ipfs_node_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED }; @@ -116,14 +120,23 @@ int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* b *bytes_written = 0; // hash if (link->hash_size > 0) { - retVal = protobuf_encode_length_delimited(1, ipfs_node_link_message_fields[0], (char*)link->hash, link->hash_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + size_t hash_length = mh_new_length(MH_H_SHA2_256, link->hash_size); + unsigned char hash[hash_length]; + mh_new(hash, MH_H_SHA2_256, link->hash, link->hash_size); + retVal = protobuf_encode_length_delimited(1, ipfs_node_link_message_fields[0], (char*)hash, hash_length, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + if (retVal == 0) { + return 0; + } + *bytes_written += bytes_used; + } + // name is optional, but they encode it anyways + if (link->name != NULL && strlen(link->name) > 0) { + retVal = protobuf_encode_string(2, ipfs_node_link_message_fields[1], link->name, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); if (retVal == 0) return 0; *bytes_written += bytes_used; - } - // name - if (link->name != NULL && strlen(link->name) > 0) { - retVal = protobuf_encode_string(2, ipfs_node_link_message_fields[1], link->name, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + } else { + retVal = protobuf_encode_string(2, ipfs_node_link_message_fields[1], "", &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); if (retVal == 0) return 0; *bytes_written += bytes_used; @@ -166,8 +179,15 @@ int ipfs_node_link_protobuf_decode(unsigned char* buffer, size_t buffer_length, pos += bytes_read; switch(field_no) { case (1): { // hash - if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&link->hash, &link->hash_size, &bytes_read) == 0) + size_t hash_size = 0; + unsigned char* hash; + if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&hash, &hash_size, &bytes_read) == 0) goto exit; + link->hash = (unsigned char*)malloc(hash_size - 2); + link->hash_size = hash_size - 2; + strncpy((char*)link->hash, (char*)hash, link->hash_size); + link->hash[hash_size-1] = 0; + free(hash); pos += bytes_read; break; } @@ -227,13 +247,6 @@ int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, si size_t bytes_used = 0; *bytes_written = 0; int retVal = 0; - // data - if (node->data_size > 0) { - retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); - if (retVal == 0) - return 0; - *bytes_written += bytes_used; - } // links struct NodeLink* current = node->head_link; while (current != NULL) { @@ -248,6 +261,13 @@ int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, si *bytes_written += bytes_used; current = current->next; } + // data + if (node->data_size > 0) { + retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + if (retVal == 0) + return 0; + *bytes_written += bytes_used; + } return 1; } @@ -369,18 +389,21 @@ int ipfs_node_set_hash(struct Node* node, const unsigned char* hash, size_t hash * Sets pointers of encoded & cached to NULL /following go method * returns 1 on success 0 on failure */ -int ipfs_node_set_data(struct Node * N, unsigned char * Data, size_t data_size) +int ipfs_node_set_data(struct Node* node, unsigned char * Data, size_t data_size) { - if(!N || !Data) + if(!node || !Data) { return 0; } - N->data = malloc(sizeof(unsigned char) * data_size); - if (N->data == NULL) + if (node->data != NULL) { + free(node->data); + } + node->data = malloc(sizeof(unsigned char) * data_size); + if (node->data == NULL) return 0; - memcpy(N->data, Data, data_size); - N->data_size = data_size; + memcpy(node->data, Data, data_size); + node->data_size = data_size; return 1; } @@ -527,17 +550,16 @@ int ipfs_node_remove_link_by_name(char * Name, struct Node * mynode) } /* ipfs_node_add_link - * Adds a link to your nodse - * @param mynode: &yournode - * @param mylink: the CID you want to create a node from - * @param linksz: sizeof(your cid here) - * Returns your node with the newly added link + * Adds a link to your node + * @param node the node to add to + * @param mylink: the link to add + * @returns true(1) on success */ -int ipfs_node_add_link(struct Node* Nl, struct NodeLink * mylink) +int ipfs_node_add_link(struct Node* node, struct NodeLink * mylink) { - if(Nl->head_link != NULL) { + if(node->head_link != NULL) { // add to existing by finding last one - struct NodeLink* current_end = Nl->head_link; + struct NodeLink* current_end = node->head_link; while(current_end->next != NULL) { current_end = current_end->next; } @@ -546,7 +568,7 @@ int ipfs_node_add_link(struct Node* Nl, struct NodeLink * mylink) } else { - Nl->head_link = mylink; + node->head_link = mylink; } return 1; } diff --git a/unixfs/unixfs.c b/unixfs/unixfs.c index 514375b..8231b84 100644 --- a/unixfs/unixfs.c +++ b/unixfs/unixfs.c @@ -48,6 +48,7 @@ int ipfs_unixfs_new(struct UnixFS** obj) { (*obj)->block_size_head = NULL; (*obj)->hash = NULL; (*obj)->hash_length = 0; + (*obj)->file_size = 0; return 1; } @@ -98,6 +99,26 @@ int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* } +int ipfs_unixfs_add_blocksize(const struct UnixFSBlockSizeNode* blocksize, struct UnixFS* unix_fs) { + struct UnixFSBlockSizeNode* last = unix_fs->block_size_head; + + if (last == NULL) { + // we're the first one + unix_fs->block_size_head = (struct UnixFSBlockSizeNode*)malloc(sizeof(struct UnixFSBlockSizeNode)); + unix_fs->block_size_head->block_size = blocksize->block_size; + } else { + // find the last one + while (last->next != NULL) { + last = last->next; + } + last->next = (struct UnixFSBlockSizeNode*)malloc(sizeof(struct UnixFSBlockSizeNode)); + last->next->block_size = blocksize->block_size; + } + + + return 1; +} + /** * Protobuf functions @@ -154,8 +175,8 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou *bytes_written += bytes_used; } // file size (optional) - if (incoming->data_type == UNIXFS_FILE && incoming->bytes_size > 0) { - retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used); + if (incoming->file_size > 0) { + retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used); if (retVal == 0) return 0; *bytes_written += bytes_used; @@ -164,6 +185,7 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou struct UnixFSBlockSizeNode* currNode = incoming->block_size_head; while (currNode != NULL) { retVal = protobuf_encode_varint(4, ipfs_unixfs_message_fields[3], currNode->block_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used); + *bytes_written += bytes_used; currNode = currNode->next; } } @@ -211,7 +233,7 @@ int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, s pos += bytes_read; break; case (3): // file size - result->bytes_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read); + result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read); pos += bytes_read; break; case (4): { // block sizes (linked list from varint)