agorise
/
c-ipfs
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Intermediate save for hash matching

yamux
jmjatlanta 2016-12-23 17:21:04 -05:00
parent 15b432c70e
commit 1dcbf8962e
6 changed files with 151 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
importer/exporter.c
View File

@ -96,7 +96,7 @@ int ipfs_exporter_to_console(const unsigned char* hash, const struct FSRepo* fs_
// process blocks
struct NodeLink* link = read_node->head_link;
printf("Links:[");
printf("{Links:[");
while (link != NULL) {
unsigned char b58[100];
ipfs_cid_hash_to_base58(link->hash, link->hash_size, b58, 100);
@ -139,9 +139,13 @@ int ipfs_exporter_object_get(int argc, char** argv) {
}
int ipfs_exporter_cat_node(struct Node* node, const struct FSRepo* fs_repo) {
// process this block, then move on to the links
for(size_t i = 0LU; i < node->data_size; i++) {
printf("%c", node->data[i]);
// process this node, then move on to the links
// build the unixfs
struct UnixFS* unix_fs;
ipfs_unixfs_protobuf_decode(node->data, node->data_size, &unix_fs);
for(size_t i = 0LU; i < unix_fs->bytes_size; i++) {
printf("%c", unix_fs->bytes[i]);
}
// process links
struct NodeLink* current = node->head_link;

62
importer/importer.c
View File

@ -13,6 +13,58 @@
* Imports OS files into the datastore
*/
/***
* adds a blocksize to the UnixFS structure stored in the data
* element of a Node
* @param node the node to work with
* @param blocksize the blocksize to add
* @returns true(1) on success
*/
int ipfs_importer_add_filesize_to_data_section(struct Node* node, size_t bytes_read) {
// now add to the data section
struct UnixFS* data_section = NULL;
if (node->data == NULL) {
// nothing in data section yet, create new UnixFS
ipfs_unixfs_new(&data_section);
data_section->data_type = UNIXFS_FILE;
} else {
ipfs_unixfs_protobuf_decode(node->data, node->data_size, &data_section);
}
struct UnixFSBlockSizeNode bs;
bs.block_size = bytes_read;
ipfs_unixfs_add_blocksize(&bs, data_section);
data_section->file_size += bytes_read;
// put the new data back in the data section
size_t protobuf_size = ipfs_unixfs_protobuf_encode_size(data_section); //delay bytes_size entry
unsigned char protobuf[protobuf_size];
ipfs_unixfs_protobuf_encode(data_section, protobuf, protobuf_size, &protobuf_size);
ipfs_unixfs_free(data_section);
ipfs_node_set_data(node, protobuf, protobuf_size);
return 1;
}
int test_hash(unsigned char* protobuf, size_t protobuf_length) {
size_t hash_size = 32;
unsigned char* hash = (unsigned char*)malloc(hash_size);
if (hash == NULL) {
return 0;
}
if (libp2p_crypto_hashing_sha256(protobuf, protobuf_length, hash) == 0) {
return 0;
}
// turn it into base58
size_t buffer_len = 100;
unsigned char buffer[buffer_len];
int retVal = ipfs_cid_hash_to_base58(hash, hash_size, buffer, buffer_len);
if (retVal == 0) {
printf("Unable to generate hash\n");
return 0;
}
printf(" hash generated from %lu: %s\n", protobuf_length, buffer);
return 1;
}
/**
* read the next chunk of bytes, create a node, and add a link to the node in the passed-in node
* @param file the file handle
@ -28,10 +80,15 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
struct Node* new_node = NULL;
struct NodeLink* new_link = NULL;
//JMJ
printf("Raw from the file");
test_hash(buffer, bytes_read);
// put the file bits into a new UnixFS file
if (ipfs_unixfs_new(&new_unixfs) == 0)
return 0;
new_unixfs->data_type = UNIXFS_FILE;
new_unixfs->file_size = bytes_read;
if (ipfs_unixfs_add_data(&buffer[0], bytes_read, new_unixfs) == 0) {
ipfs_unixfs_free(new_unixfs);
return 0;
@ -48,6 +105,9 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
ipfs_unixfs_free(new_unixfs);
return 0;
}
//JMJ
printf("unixfs protobuf");
test_hash(protobuf, protobuf_size);
// we're done with the UnixFS object
ipfs_unixfs_free(new_unixfs);
@ -76,6 +136,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
ipfs_node_free(new_node);
return 0;
}
ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read);
ipfs_node_free(new_node);
} else {
// if there are no existing links, put what we pulled from the file into parent_node
@ -107,6 +168,7 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
ipfs_node_free(new_node);
return 0;
}
ipfs_importer_add_filesize_to_data_section(parent_node, bytes_read);
ipfs_node_free(new_node);
}
// persist the main node

4
include/ipfs/unixfs/unixfs.h
View File

@ -43,7 +43,7 @@ struct UnixFS {
enum UnixFSDataType data_type;
size_t bytes_size; // the size of the bytes array
unsigned char* bytes; // an array of bytes
// size_t file_size; // the file size - I mimick this one
size_t file_size; // when saving files that have been chunked
struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes
unsigned char* hash; // not saved
size_t hash_length; // not saved
@ -76,6 +76,8 @@ int ipfs_unixfs_free(struct UnixFS* obj);
*/
int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs);
int ipfs_unixfs_add_blocksize(const struct UnixFSBlockSizeNode* blocksize, struct UnixFS* unix_fs);
/**
* Protobuf functions
*/

2
merkledag/merkledag.c
View File

@ -6,6 +6,8 @@
#include <string.h>
#include "libp2p/crypto/sha256.h"
#include "mh/multihash.h"
#include "mh/hashes.h"
#include "ipfs/merkledag/merkledag.h"
#include "ipfs/unixfs/unixfs.h"

80
merkledag/node.c
View File

@ -6,9 +6,13 @@
#include <stdio.h>
#include <string.h>
#include "inttypes.h"
#include "ipfs/cid/cid.h"
#include "mh/multihash.h"
#include "mh/hashes.h"
#include "ipfs/cid/cid.h"
#include "ipfs/merkledag/node.h"
#include "ipfs/unixfs/unixfs.h"
// for protobuf Node (all fields optional) data (optional bytes) links (repeated node_link)
enum WireType ipfs_node_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED };
@ -116,14 +120,23 @@ int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* b
*bytes_written = 0;
// hash
if (link->hash_size > 0) {
retVal = protobuf_encode_length_delimited(1, ipfs_node_link_message_fields[0], (char*)link->hash, link->hash_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
size_t hash_length = mh_new_length(MH_H_SHA2_256, link->hash_size);
unsigned char hash[hash_length];
mh_new(hash, MH_H_SHA2_256, link->hash, link->hash_size);
retVal = protobuf_encode_length_delimited(1, ipfs_node_link_message_fields[0], (char*)hash, hash_length, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
if (retVal == 0) {
return 0;
}
*bytes_written += bytes_used;
}
// name is optional, but they encode it anyways
if (link->name != NULL && strlen(link->name) > 0) {
retVal = protobuf_encode_string(2, ipfs_node_link_message_fields[1], link->name, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
}
// name
if (link->name != NULL && strlen(link->name) > 0) {
retVal = protobuf_encode_string(2, ipfs_node_link_message_fields[1], link->name, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
} else {
retVal = protobuf_encode_string(2, ipfs_node_link_message_fields[1], "", &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
@ -166,8 +179,15 @@ int ipfs_node_link_protobuf_decode(unsigned char* buffer, size_t buffer_length,
pos += bytes_read;
switch(field_no) {
case (1): { // hash
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&link->hash, &link->hash_size, &bytes_read) == 0)
size_t hash_size = 0;
unsigned char* hash;
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&hash, &hash_size, &bytes_read) == 0)
goto exit;
link->hash = (unsigned char*)malloc(hash_size - 2);
link->hash_size = hash_size - 2;
strncpy((char*)link->hash, (char*)hash, link->hash_size);
link->hash[hash_size-1] = 0;
free(hash);
pos += bytes_read;
break;
}
@ -227,13 +247,6 @@ int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, si
size_t bytes_used = 0;
*bytes_written = 0;
int retVal = 0;
// data
if (node->data_size > 0) {
retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
}
// links
struct NodeLink* current = node->head_link;
while (current != NULL) {
@ -248,6 +261,13 @@ int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, si
*bytes_written += bytes_used;
current = current->next;
}
// data
if (node->data_size > 0) {
retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
}
return 1;
}
@ -369,18 +389,21 @@ int ipfs_node_set_hash(struct Node* node, const unsigned char* hash, size_t hash
* Sets pointers of encoded & cached to NULL /following go method
* returns 1 on success 0 on failure
*/
int ipfs_node_set_data(struct Node * N, unsigned char * Data, size_t data_size)
int ipfs_node_set_data(struct Node* node, unsigned char * Data, size_t data_size)
{
if(!N || !Data)
if(!node || !Data)
{
return 0;
}
N->data = malloc(sizeof(unsigned char) * data_size);
if (N->data == NULL)
if (node->data != NULL) {
free(node->data);
}
node->data = malloc(sizeof(unsigned char) * data_size);
if (node->data == NULL)
return 0;
memcpy(N->data, Data, data_size);
N->data_size = data_size;
memcpy(node->data, Data, data_size);
node->data_size = data_size;
return 1;
}
@ -527,17 +550,16 @@ int ipfs_node_remove_link_by_name(char * Name, struct Node * mynode)
}
/* ipfs_node_add_link
* Adds a link to your nodse
* @param mynode: &yournode
* @param mylink: the CID you want to create a node from
* @param linksz: sizeof(your cid here)
* Returns your node with the newly added link
* Adds a link to your node
* @param node the node to add to
* @param mylink: the link to add
* @returns true(1) on success
*/
int ipfs_node_add_link(struct Node* Nl, struct NodeLink * mylink)
int ipfs_node_add_link(struct Node* node, struct NodeLink * mylink)
{
if(Nl->head_link != NULL) {
if(node->head_link != NULL) {
// add to existing by finding last one
struct NodeLink* current_end = Nl->head_link;
struct NodeLink* current_end = node->head_link;
while(current_end->next != NULL) {
current_end = current_end->next;
}
@ -546,7 +568,7 @@ int ipfs_node_add_link(struct Node* Nl, struct NodeLink * mylink)
}
else
{
Nl->head_link = mylink;
node->head_link = mylink;
}
return 1;
}

28
unixfs/unixfs.c
View File

@ -48,6 +48,7 @@ int ipfs_unixfs_new(struct UnixFS** obj) {
(*obj)->block_size_head = NULL;
(*obj)->hash = NULL;
(*obj)->hash_length = 0;
(*obj)->file_size = 0;
return 1;
}
@ -98,6 +99,26 @@ int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS*
}
int ipfs_unixfs_add_blocksize(const struct UnixFSBlockSizeNode* blocksize, struct UnixFS* unix_fs) {
struct UnixFSBlockSizeNode* last = unix_fs->block_size_head;
if (last == NULL) {
// we're the first one
unix_fs->block_size_head = (struct UnixFSBlockSizeNode*)malloc(sizeof(struct UnixFSBlockSizeNode));
unix_fs->block_size_head->block_size = blocksize->block_size;
} else {
// find the last one
while (last->next != NULL) {
last = last->next;
}
last->next = (struct UnixFSBlockSizeNode*)malloc(sizeof(struct UnixFSBlockSizeNode));
last->next->block_size = blocksize->block_size;
}
return 1;
}
/**
* Protobuf functions
@ -154,8 +175,8 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou
*bytes_written += bytes_used;
}
// file size (optional)
if (incoming->data_type == UNIXFS_FILE && incoming->bytes_size > 0) {
retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
if (incoming->file_size > 0) {
retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
@ -164,6 +185,7 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou
struct UnixFSBlockSizeNode* currNode = incoming->block_size_head;
while (currNode != NULL) {
retVal = protobuf_encode_varint(4, ipfs_unixfs_message_fields[3], currNode->block_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
*bytes_written += bytes_used;
currNode = currNode->next;
}
}
@ -211,7 +233,7 @@ int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, s
pos += bytes_read;
break;
case (3): // file size
result->bytes_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
pos += bytes_read;
break;
case (4): { // block sizes (linked list from varint)