diff --git a/Makefile b/Makefile index 9cb12f2..3281c2e 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ all: cd flatfs; make all; cd datastore; make all; cd thirdparty; make all; + cd unixfs; make all; cd main; make all; cd test; make all; @@ -33,6 +34,7 @@ clean: cd flatfs; make clean; cd datastore; make clean; cd thirdparty; make clean; + cd unixfs; make clean; cd main; make clean; cd test; make clean; diff --git a/blocks/blockstore.c b/blocks/blockstore.c index bbe4157..0dc1b29 100644 --- a/blocks/blockstore.c +++ b/blocks/blockstore.c @@ -40,12 +40,12 @@ unsigned char* ipfs_blockstore_cid_to_base32(const struct Cid* cid) { return buffer; } -unsigned char* ipfs_blockstore_hash_to_base32(const unsigned char* hash, size_t hash_size) { - size_t key_length = libp2p_crypto_encoding_base32_encode_size(hash_size); +unsigned char* ipfs_blockstore_hash_to_base32(const unsigned char* hash, size_t hash_length) { + size_t key_length = libp2p_crypto_encoding_base32_encode_size(hash_length); unsigned char* buffer = (unsigned char*)malloc(key_length + 1); if (buffer == NULL) return NULL; - int retVal = ipfs_datastore_helper_ds_key_from_binary(hash, hash_size, &buffer[0], key_length, &key_length); + int retVal = ipfs_datastore_helper_ds_key_from_binary(hash, hash_length, &buffer[0], key_length, &key_length); if (retVal == 0) { free(buffer); return NULL; @@ -145,3 +145,135 @@ int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) { free(filename); return 1; } + +/*** + * Put a struct UnixFS in the blockstore + * @param unix_fs the structure + * @param fs_repo the repo to place the strucure in + * @param bytes_written the number of bytes written to the blockstore + * @returns true(1) on success + */ +int ipfs_blockstore_put_unixfs(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written) { + // from blockstore.go line 118 + int retVal = 0; + + // Get Datastore key, which is a base32 key of the multihash, + unsigned char* key = ipfs_blockstore_hash_to_base32(unix_fs->hash, unix_fs->hash_length); + if (key == NULL) { + free(key); + return 0; + } + + //TODO: put this in subdirectories + + // turn the block into a binary array + size_t protobuf_len = ipfs_unixfs_protobuf_encode_size(unix_fs); + unsigned char protobuf[protobuf_len]; + retVal = ipfs_unixfs_protobuf_encode(unix_fs, protobuf, protobuf_len, &protobuf_len); + if (retVal == 0) { + free(key); + return 0; + } + + // now write byte array to file + char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key); + if (filename == NULL) { + free(key); + return 0; + } + + FILE* file = fopen(filename, "wb"); + *bytes_written = fwrite(protobuf, 1, protobuf_len, file); + fclose(file); + if (*bytes_written != protobuf_len) { + free(key); + free(filename); + return 0; + } + + // send to Put with key (this is now done separately) + //fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore); + + free(key); + free(filename); + return 1; +} + +/*** + * Find a UnixFS struct based on its hash + * @param hash the hash to look for + * @param hash_length the length of the hash + * @param unix_fs the struct to fill + * @param fs_repo where to look for the data + * @returns true(1) on success + */ +int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, struct UnixFS** block, const struct FSRepo* fs_repo) { + // get datastore key, which is a base32 key of the multihash + unsigned char* key = ipfs_blockstore_hash_to_base32(hash, hash_length); + + char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key); + + size_t file_size = os_utils_file_size(filename); + unsigned char buffer[file_size]; + + FILE* file = fopen(filename, "rb"); + size_t bytes_read = fread(buffer, 1, file_size, file); + fclose(file); + + int retVal = ipfs_unixfs_protobuf_decode(buffer, bytes_read, block); + + free(key); + free(filename); + + return retVal; +} + +/*** + * Put a struct Node in the blockstore + * @param node the structure + * @param fs_repo the repo to place the strucure in + * @param bytes_written the number of bytes written to the blockstore + * @returns true(1) on success + */ +int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written) { + // from blockstore.go line 118 + int retVal = 0; + + // Get Datastore key, which is a base32 key of the multihash, + unsigned char* key = ipfs_blockstore_hash_to_base32(node->hash, node->hash_size); + if (key == NULL) { + free(key); + return 0; + } + + //TODO: put this in subdirectories + + // turn the block into a binary array + size_t protobuf_len = ipfs_node_protobuf_encode_size(node); + unsigned char protobuf[protobuf_len]; + retVal = ipfs_node_protobuf_encode(node, protobuf, protobuf_len, &protobuf_len); + if (retVal == 0) { + free(key); + return 0; + } + + // now write byte array to file + char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key); + if (filename == NULL) { + free(key); + return 0; + } + + FILE* file = fopen(filename, "wb"); + *bytes_written = fwrite(protobuf, 1, protobuf_len, file); + fclose(file); + if (*bytes_written != protobuf_len) { + free(key); + free(filename); + return 0; + } + + free(key); + free(filename); + return 1; +} diff --git a/importer/importer.c b/importer/importer.c index d58cda0..e8977e3 100644 --- a/importer/importer.c +++ b/importer/importer.c @@ -23,27 +23,43 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs unsigned char buffer[MAX_DATA_SIZE]; size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file); + // put the file bits into a new UnixFS file + struct UnixFS* new_unixfs = NULL; + ipfs_unixfs_new(&new_unixfs); + new_unixfs->data_type = UNIXFS_FILE; + ipfs_unixfs_add_data(&buffer[0], bytes_read, new_unixfs); + // protobuf the UnixFS + size_t protobuf_size = ipfs_unixfs_protobuf_encode_size(new_unixfs); + unsigned char protobuf[protobuf_size]; + size_t bytes_written = 0; + ipfs_unixfs_protobuf_encode(new_unixfs, protobuf, protobuf_size, &bytes_written); + // we're done with the object + ipfs_unixfs_free(new_unixfs); // create a new node struct Node* new_node = NULL; - ipfs_node_new_from_data(buffer, bytes_read, &new_node); + ipfs_node_new_from_data(protobuf, bytes_written, &new_node); + ipfs_node_set_hash(new_node, new_unixfs->hash, new_unixfs->hash_length); // persist - ipfs_merkledag_add(new_node, fs_repo); + size_t size_of_node = 0; + ipfs_merkledag_add(new_node, fs_repo, &size_of_node); // put link in parent node struct NodeLink* new_link = NULL; ipfs_node_link_create("", new_node->hash, new_node->hash_size, &new_link); - new_link->t_size = new_node->data_size; + new_link->t_size = size_of_node; *total_size += new_link->t_size; ipfs_node_add_link(parent_node, new_link); ipfs_node_free(new_node); - // save the parent_node if it is time... if (bytes_read != MAX_DATA_SIZE) { - // build UnixFS for file + // We have read everything, now save the parent_node, + // which is a sort of "directory" entry + /* + // build UnixFS for the parent struct UnixFS* unix_fs; if (ipfs_unixfs_new(&unix_fs) == 0) { return 0; } unix_fs->data_type = UNIXFS_FILE; - unix_fs->file_size = *total_size; + unix_fs->bytes_size = *total_size; // now encode unixfs and put in parent_node->data size_t temp_size = ipfs_unixfs_protobuf_encode_size(unix_fs); unsigned char temp[temp_size]; @@ -59,8 +75,10 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs return 0; } memcpy(parent_node->data, temp, bytes_written); + ipfs_unixfs_free(unix_fs); + */ // persist the main node - ipfs_merkledag_add(parent_node, fs_repo); + ipfs_merkledag_add(parent_node, fs_repo, &bytes_written); } return bytes_read; } diff --git a/include/ipfs/blocks/blockstore.h b/include/ipfs/blocks/blockstore.h index f209c42..77f4b00 100644 --- a/include/ipfs/blocks/blockstore.h +++ b/include/ipfs/blocks/blockstore.h @@ -37,5 +37,28 @@ int ipfs_blockstore_get(const unsigned char* hash, size_t hash_length, struct Bl */ int ipfs_blockstore_put(struct Block* block, const struct FSRepo* fs_repo); +/*** + * Put a struct UnixFS in the blockstore + * @param unix_fs the structure + * @param fs_repo the repo to place the strucure in + * @param bytes_written the number of bytes written to the blockstore + * @returns true(1) on success + */ +int ipfs_blockstore_put_unixfs(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written); + +/*** + * Find a UnixFS struct based on its hash + * @param hash the hash to look for + * @param hash_length the length of the hash + * @param unix_fs the struct to fill + * @param fs_repo where to look for the data + * @returns true(1) on success + */ +int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, struct UnixFS** block, const struct FSRepo* fs_repo); + +/** + * Put a struct Node in the blockstore + */ +int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written); #endif diff --git a/include/ipfs/merkledag/merkledag.h b/include/ipfs/merkledag/merkledag.h index 10adcca..189ace1 100644 --- a/include/ipfs/merkledag/merkledag.h +++ b/include/ipfs/merkledag/merkledag.h @@ -10,10 +10,11 @@ /*** * Adds a node to the dagService and blockService * @param node the node to add - * @param cid the resultant cid that was added + * @param fs_repo the repo to add to + * @param bytes_written the number of bytes written * @returns true(1) on success */ -int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo); +int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_written); /*** * Retrieves a node from the datastore based on the cid diff --git a/include/ipfs/merkledag/node.h b/include/ipfs/merkledag/node.h index 3a390df..fe6469b 100644 --- a/include/ipfs/merkledag/node.h +++ b/include/ipfs/merkledag/node.h @@ -74,7 +74,7 @@ int ipfs_node_link_free(struct NodeLink * node_link); * @param link the link to examine * @returns the maximum size that should be needed */ -size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link); +size_t ipfs_node_link_protobuf_encode_size(const struct NodeLink* link); /*** * Encode a NodeLink into protobuf format @@ -84,7 +84,7 @@ size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link); * @pram bytes_written the amount of the buffer used * @returns true(1) on success */ -int ipfs_node_link_protobuf_encode(struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written); +int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written); /**** * Decode from a byte array into a NodeLink @@ -102,7 +102,7 @@ int ipfs_node_link_protobuf_decode(unsigned char* buffer, size_t buffer_length, * @param node the node to examine * @returns the max size of an encoded stream of bytes, if it were encoded */ -size_t ipfs_node_protobuf_encode_size(struct Node* node); +size_t ipfs_node_protobuf_encode_size(const struct Node* node); /*** * Encode a node into a protobuf byte stream @@ -112,7 +112,7 @@ size_t ipfs_node_protobuf_encode_size(struct Node* node); * @param bytes_written how much of buffer was used * @returns true(1) on success */ -int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written); +int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written); /*** * Decode a stream of bytes into a Node structure diff --git a/include/ipfs/repo/fsrepo/fs_repo.h b/include/ipfs/repo/fsrepo/fs_repo.h index 4d019e8..cc6d834 100644 --- a/include/ipfs/repo/fsrepo/fs_repo.h +++ b/include/ipfs/repo/fsrepo/fs_repo.h @@ -5,8 +5,10 @@ #define fs_repo_h #include -#include "ipfs/repo/config/config.h" +#include "ipfs/repo/config/config.h" +#include "ipfs/unixfs/unixfs.h" +#include "ipfs/merkledag/node.h" /** * a structure to hold the repo info @@ -73,4 +75,23 @@ int ipfs_repo_fsrepo_init(struct FSRepo* config); int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo); int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo); +/*** + * Write a unixfs to the datastore and blockstore + * @param unix_fs the struct to write + * @param fs_repo the repo to write to + * @returns true(1) on success + */ +int ipfs_repo_fsrepo_unixfs_write(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written); +int ipfs_repo_fsrepo_unixfs_read(const unsigned char* hash, size_t hash_length, struct UnixFS** unix_fs, const struct FSRepo* fs_repo); + +/*** + * Write a struct Node to the datastore and blockstore + * @param node the struct to write + * @param fs_repo the repo to write to + * @returns true(1) on success + */ +int ipfs_repo_fsrepo_node_write(const struct Node* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written); +int ipfs_repo_fsrepo_node_read(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo); + + #endif /* fs_repo_h */ diff --git a/include/ipfs/unixfs/format.h b/include/ipfs/unixfs/format.h index 8d3d33f..db7b96b 100644 --- a/include/ipfs/unixfs/format.h +++ b/include/ipfs/unixfs/format.h @@ -1,4 +1,4 @@ -enum UnixFSFormatType { RAW, FILE, DIRECTORY, METADATA, SYMLINK }; +enum UnixFSFormatType { RAW, UNIXFS_FILE, DIRECTORY, METADATA, SYMLINK }; struct UnixFSData { enum UnixFSFormatType type; diff --git a/include/ipfs/unixfs/unixfs.h b/include/ipfs/unixfs/unixfs.h index e1c737b..e08b2f5 100644 --- a/include/ipfs/unixfs/unixfs.h +++ b/include/ipfs/unixfs/unixfs.h @@ -43,8 +43,10 @@ struct UnixFS { enum UnixFSDataType data_type; size_t bytes_size; // the size of the bytes array unsigned char* bytes; // an array of bytes - size_t file_size; // the file size + // size_t file_size; // the file size - I mimick this one struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes + unsigned char* hash; // not saved + size_t hash_length; // not saved }; struct UnixFSMetaData { @@ -65,6 +67,15 @@ int ipfs_unixfs_new(struct UnixFS** obj); */ int ipfs_unixfs_free(struct UnixFS* obj); +/*** + * Write data to data section of a UnixFS stuct. NOTE: this also calculates a sha256 hash + * @param data the data to write + * @param data_length the length of the data + * @param unix_fs the struct to add to + * @returns true(1) on success + */ +int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs); + /** * Protobuf functions */ @@ -74,7 +85,7 @@ int ipfs_unixfs_free(struct UnixFS* obj); * @param obj what will be encoded * @returns the size of the buffer necessary to encode the object */ -size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj); +size_t ipfs_unixfs_protobuf_encode_size(const struct UnixFS* obj); /*** * Encode a UnixFS object into protobuf format diff --git a/main/Makefile b/main/Makefile index 256b9a9..5e2ee2c 100644 --- a/main/Makefile +++ b/main/Makefile @@ -9,14 +9,14 @@ OBJS = main.o \ ../commands/argument.o ../commands/command_option.o ../commands/command.o ../commands/cli/parse.o \ ../core/builder.o \ ../datastore/ds_helper.o \ + ../dnslink/dnslink.o \ ../flatfs/flatfs.o \ ../importer/importer.o \ ../importer/exporter.o \ - ../dnslink/dnslink.o \ ../path/path.o \ - ../namesys/isdomain.o \ ../merkledag/merkledag.o ../merkledag/node.o \ ../multibase/multibase.o \ + ../namesys/isdomain.o \ ../os/utils.o \ ../repo/init.o \ ../repo/fsrepo/fs_repo.o ../repo/fsrepo/jsmn.o ../repo/fsrepo/lmdb_datastore.o \ diff --git a/main/ipfs b/main/ipfs deleted file mode 100755 index 223aa74..0000000 Binary files a/main/ipfs and /dev/null differ diff --git a/merkledag/merkledag.c b/merkledag/merkledag.c index 37bf4bc..32ce78c 100644 --- a/merkledag/merkledag.c +++ b/merkledag/merkledag.c @@ -1,41 +1,46 @@ /** * A basic storage building block of the IPFS system */ +#include +#include +#include +#include "libp2p/crypto/sha256.h" #include "ipfs/merkledag/merkledag.h" +#include "ipfs/unixfs/unixfs.h" /*** * Adds a node to the dagService and blockService * @param node the node to add - * @param cid the resultant cid that was added + * @param fs_repo the repo to add to + * @param bytes_written the number of bytes written * @returns true(1) on success */ -int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) { +int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_written) { // taken from merkledag.go line 59 + int retVal = 0; - struct Block* block = NULL; + // compute the hash + size_t protobuf_size = ipfs_node_protobuf_encode_size(node); + unsigned char protobuf[protobuf_size]; + size_t bytes_encoded; + retVal = ipfs_node_protobuf_encode(node, protobuf, protobuf_size, &bytes_encoded); - // protobuf the node - size_t protobuf_len = ipfs_node_protobuf_encode_size(node); - size_t bytes_written = 0; - unsigned char protobuf[protobuf_len]; - ipfs_node_protobuf_encode(node, protobuf, protobuf_len, &bytes_written); - - // turn the node into a block - ipfs_blocks_block_new(&block); - ipfs_blocks_block_add_data(protobuf, bytes_written, block); - - // write to block store & datastore - int retVal = ipfs_repo_fsrepo_block_write(block, fs_repo); - if (retVal == 0) { - ipfs_blocks_block_free(block); + node->hash_size = 32; + node->hash = (unsigned char*)malloc(node->hash_size); + if (node->hash == NULL) { + return 0; + } + if (libp2p_crypto_hashing_sha256(protobuf, bytes_encoded, &node->hash[0]) == 0) { + free(node->hash); return 0; } - ipfs_node_set_hash(node, block->cid->hash, block->cid->hash_length); - - if (block != NULL) - ipfs_blocks_block_free(block); + // write to block store & datastore + retVal = ipfs_repo_fsrepo_node_write(node, fs_repo, bytes_written); + if (retVal == 0) { + return 0; + } // TODO: call HasBlock (unsure why as yet) return 1; @@ -51,7 +56,8 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) { */ int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node** node, const struct FSRepo* fs_repo) { int retVal = 1; - struct Block* block; + //struct Block* block; + struct UnixFS* unix_fs; size_t key_length = 100; unsigned char key[key_length]; @@ -61,27 +67,27 @@ int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node* if (retVal == 0) return 0; - // we have the record from the db. Go get the block from the blockstore - retVal = ipfs_repo_fsrepo_block_read(hash, hash_size, &block, fs_repo); + // we have the record from the db. Go get the UnixFS from the blockstore + retVal = ipfs_repo_fsrepo_unixfs_read(hash, hash_size, &unix_fs, fs_repo); if (retVal == 0) { return 0; } // now convert the block into a node - if (ipfs_node_protobuf_decode(block->data, block->data_length, node) == 0) { - ipfs_blocks_block_free(block); + if (ipfs_node_protobuf_decode(unix_fs->bytes, unix_fs->bytes_size, node) == 0) { + ipfs_unixfs_free(unix_fs); return 0; } // set the cid on the node if (ipfs_node_set_hash(*node, hash, hash_size) == 0) { - ipfs_blocks_block_free(block); + ipfs_unixfs_free(unix_fs); ipfs_node_free(*node); return 0; } // free resources - ipfs_blocks_block_free(block); + ipfs_unixfs_free(unix_fs); return 1; } diff --git a/merkledag/node.c b/merkledag/node.c index ddb3e44..8314c26 100644 --- a/merkledag/node.c +++ b/merkledag/node.c @@ -10,7 +10,7 @@ #include "ipfs/merkledag/node.h" -// for protobuf Node (all fields optional) links (repeated node_link) data (optional bytes) +// for protobuf Node (all fields optional) data (optional bytes) links (repeated node_link) enum WireType ipfs_node_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED }; // for protobuf NodeLink (all fields optional) hash name tsize enum WireType ipfs_node_link_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED, WIRETYPE_VARINT }; @@ -84,7 +84,7 @@ int ipfs_node_link_free(struct NodeLink * node_link) * @param link the link to examine * @returns the maximum length of the encoded NodeLink */ -size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link) { +size_t ipfs_node_link_protobuf_encode_size(const struct NodeLink* link) { if (link == NULL) return 0; // hash, name, tsize @@ -109,7 +109,7 @@ size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link) { * @param bytes_written the number of bytes written to buffer * @returns true(1) on success */ -int ipfs_node_link_protobuf_encode(struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) { +int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) { // 3 fields, hash (length delimited), name (length delimited), tsize (varint) size_t bytes_used = 0; int retVal = 0; @@ -200,7 +200,7 @@ exit: /*** * return an approximate size of the encoded node */ -size_t ipfs_node_protobuf_encode_size(struct Node* node) { +size_t ipfs_node_protobuf_encode_size(const struct Node* node) { size_t size = 0; // links struct NodeLink* current = node->head_link; @@ -223,10 +223,17 @@ size_t ipfs_node_protobuf_encode_size(struct Node* node) { * @param bytes_written how much of buffer was used * @returns true(1) on success */ -int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) { +int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) { size_t bytes_used = 0; *bytes_written = 0; int retVal = 0; + // data + if (node->data_size > 0) { + retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + if (retVal == 0) + return 0; + *bytes_written += bytes_used; + } // links struct NodeLink* current = node->head_link; while (current != NULL) { @@ -235,19 +242,12 @@ int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t m retVal = ipfs_node_link_protobuf_encode(current, temp, temp_size, &bytes_used); if (retVal == 0) return 0; - retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)temp, bytes_used, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); + retVal = protobuf_encode_length_delimited(2, ipfs_node_message_fields[1], (char*)temp, bytes_used, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); if (retVal == 0) return 0; *bytes_written += bytes_used; current = current->next; } - // data - if (node->data_size > 0) { - retVal = protobuf_encode_length_delimited(2, ipfs_node_message_fields[1], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); - if (retVal == 0) - return 0; - *bytes_written += bytes_used; - } return 1; } @@ -261,8 +261,8 @@ int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t m */ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struct Node** node) { /* - * Field 0: link * Field 1: data + * Field 2: link */ size_t pos = 0; int retVal = 0; @@ -282,7 +282,13 @@ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struc } pos += bytes_read; switch(field_no) { - case (1): {// links + case (1): { // data + if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*node)->data), &((*node)->data_size), &bytes_read) == 0) + goto exit; + pos += bytes_read; + break; + } + case (2): {// links if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0) goto exit; pos += bytes_read; @@ -293,12 +299,6 @@ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struc ipfs_node_add_link(*node, temp_link); break; } - case (2): { // data - if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*node)->data), &((*node)->data_size), &bytes_read) == 0) - goto exit; - pos += bytes_read; - break; - } } } diff --git a/repo/fsrepo/fs_repo.c b/repo/fsrepo/fs_repo.c index 2d4d61f..5e4d8e4 100644 --- a/repo/fsrepo/fs_repo.c +++ b/repo/fsrepo/fs_repo.c @@ -554,6 +554,69 @@ int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_re return 1; } +/*** + * Write a unixfs to the datastore and blockstore + * @param unix_fs the struct to write + * @param fs_repo the repo to write to + * @param bytes_written number of bytes written to the repo + * @returns true(1) on success + */ +int ipfs_repo_fsrepo_unixfs_write(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written) { + /** + * What is put in the blockstore is the block. + * What is put in the datastore is the multihash (the Cid) as the key, + * and the base32 encoded multihash as the value. + */ + int retVal = 1; + retVal = ipfs_blockstore_put_unixfs(unix_fs, fs_repo, bytes_written); + if (retVal == 0) + return 0; + // take the hash, base32 it, and send both to the datastore + size_t fs_key_length = 100; + unsigned char fs_key[fs_key_length]; + retVal = ipfs_datastore_helper_ds_key_from_binary(unix_fs->hash, unix_fs->hash_length, fs_key, fs_key_length, &fs_key_length); + if (retVal == 0) + return 0; + retVal = fs_repo->config->datastore->datastore_put(unix_fs->hash, unix_fs->hash_length, fs_key, fs_key_length, fs_repo->config->datastore); + if (retVal == 0) + return 0; + return 1; +} + +/*** + * Write a unixfs to the datastore and blockstore + * @param unix_fs the struct to write + * @param fs_repo the repo to write to + * @param bytes_written number of bytes written to the repo + * @returns true(1) on success + */ +int ipfs_repo_fsrepo_node_write(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written) { + /** + * What is put in the blockstore is the node. + * What is put in the datastore is the multihash as the key, + * and the base32 encoded multihash as the value. + */ + int retVal = 1; + retVal = ipfs_blockstore_put_node(node, fs_repo, bytes_written); + if (retVal == 0) + return 0; + // take the hash, base32 it, and send both to the datastore + size_t fs_key_length = 100; + unsigned char fs_key[fs_key_length]; + retVal = ipfs_datastore_helper_ds_key_from_binary(node->hash, node->hash_size, fs_key, fs_key_length, &fs_key_length); + if (retVal == 0) + return 0; + retVal = fs_repo->config->datastore->datastore_put(node->hash, node->hash_size, fs_key, fs_key_length, fs_repo->config->datastore); + if (retVal == 0) + return 0; + return 1; +} + +int ipfs_repo_fsrepo_node_read(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo) { + return 0; +} + + int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo) { int retVal = 0; @@ -569,3 +632,18 @@ int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, s return retVal; } +int ipfs_repo_fsrepo_unixfs_read(const unsigned char* hash, size_t hash_length, struct UnixFS** unix_fs, const struct FSRepo* fs_repo) { + int retVal = 0; + + // get the base32 hash from the database + // We do this only to see if it is in the database + size_t fs_key_length = 100; + unsigned char fs_key[fs_key_length]; + retVal = fs_repo->config->datastore->datastore_get((const char*)hash, hash_length, fs_key, fs_key_length, &fs_key_length, fs_repo->config->datastore); + if (retVal == 0) // maybe it doesn't exist? + return 0; + // now get the block from the blockstore + retVal = ipfs_blockstore_get_unixfs(hash, hash_length, unix_fs, fs_repo); + return retVal; +} + diff --git a/test/merkledag/test_merkledag.h b/test/merkledag/test_merkledag.h index 46feb46..22286df 100644 --- a/test/merkledag/test_merkledag.h +++ b/test/merkledag/test_merkledag.h @@ -42,8 +42,8 @@ int test_merkledag_get_data() { // create a node struct Node* node1; retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1); - - retVal = ipfs_merkledag_add(node1, fs_repo); + size_t bytes_written = 0; + retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written); if (retVal == 0) { ipfs_node_free(node1); ipfs_repo_fsrepo_free(fs_repo); @@ -105,7 +105,8 @@ int test_merkledag_add_data() { struct Node* node1; retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1); - retVal = ipfs_merkledag_add(node1, fs_repo); + size_t bytes_written = 0; + retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written); if (retVal == 0) { ipfs_node_free(node1); return 0; @@ -124,7 +125,7 @@ int test_merkledag_add_data() { // adding the same binary again should do nothing (the hash should be the same) struct Node* node2; retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node2); - retVal = ipfs_merkledag_add(node2, fs_repo); + retVal = ipfs_merkledag_add(node2, fs_repo, &bytes_written); if (retVal == 0) { ipfs_node_free(node1); ipfs_node_free(node2); @@ -160,7 +161,7 @@ int test_merkledag_add_data() { struct Node* node3; retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node3); - retVal = ipfs_merkledag_add(node3, fs_repo); + retVal = ipfs_merkledag_add(node3, fs_repo, &bytes_written); if (retVal == 0) { ipfs_node_free(node1); ipfs_node_free(node2); @@ -207,7 +208,8 @@ int test_merkledag_add_node() { return 0; } - retVal = ipfs_merkledag_add(node1, fs_repo); + size_t bytes_written = 0; + retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written); if (retVal == 0) { ipfs_repo_fsrepo_free(fs_repo); ipfs_node_free(node1); @@ -250,7 +252,8 @@ int test_merkledag_add_node_with_links() { return 0; } - retVal = ipfs_merkledag_add(node1, fs_repo); + size_t bytes_written = 0; + retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written); if (retVal == 0) { ipfs_repo_fsrepo_free(fs_repo); ipfs_node_free(node1); diff --git a/test/storage/test_unixfs.h b/test/storage/test_unixfs.h index 1ba5acf..c1ab839 100644 --- a/test/storage/test_unixfs.h +++ b/test/storage/test_unixfs.h @@ -65,7 +65,6 @@ int test_unixfs_encode_smallfile() { memcpy(unixfs->bytes, bytes, 35); unixfs->bytes_size = 35; unixfs->data_type = UNIXFS_FILE; - unixfs->file_size = 35; size_t protobuf_size = 43; unsigned char protobuf[protobuf_size]; diff --git a/unixfs/Makefile b/unixfs/Makefile new file mode 100644 index 0000000..07fb836 --- /dev/null +++ b/unixfs/Makefile @@ -0,0 +1,18 @@ +CC = gcc +CFLAGS = -O0 -I../include -I../../c-libp2p/include -I../../c-multihash/include -I../../c-multiaddr/include -I../../c-protobuf -Wall + +ifdef DEBUG +CFLAGS += -g3 +endif + +LFLAGS = +DEPS = +OBJS = unixfs.o + +%.o: %.c $(DEPS) + $(CC) -c -o $@ $< $(CFLAGS) + +all: $(OBJS) + +clean: + rm -f *.o diff --git a/unixfs/format.c b/unixfs/format.c.old similarity index 88% rename from unixfs/format.c rename to unixfs/format.c.old index 3a448a5..42c55da 100644 --- a/unixfs/format.c +++ b/unixfs/format.c.old @@ -1,7 +1,10 @@ #include +#include +#include #include "ipfs/unixfs/format.h" + /** * Copy data into a new struct * @param type the type of data @@ -12,7 +15,7 @@ */ int ipfs_unixfs_format_new(enum UnixFSFormatType type, const unsigned char* data, size_t data_length, struct UnixFSData** result) { - struct UnixFSData* new_struct = malloc(sizeof(struct UnixFSData)); + struct UnixFSData* new_struct = (struct UnixFSData*)malloc(sizeof(struct UnixFSData)); if (new_struct == NULL) return 0; diff --git a/unixfs/unixfs.c b/unixfs/unixfs.c index a31be6e..acb9420 100644 --- a/unixfs/unixfs.c +++ b/unixfs/unixfs.c @@ -4,7 +4,9 @@ #include #include +#include +#include "libp2p/crypto/sha256.h" #include "ipfs/unixfs/unixfs.h" #include "protobuf.h" #include "varint.h" @@ -43,8 +45,9 @@ int ipfs_unixfs_new(struct UnixFS** obj) { (*obj)->bytes = 0; (*obj)->bytes_size = 0; (*obj)->data_type = UNIXFS_RAW; - (*obj)->file_size = 0; (*obj)->block_size_head = NULL; + (*obj)->hash = NULL; + (*obj)->hash_length = 0; return 1; } @@ -56,6 +59,40 @@ int ipfs_unixfs_free(struct UnixFS* obj) { return 1; } +/*** + * Write data to data section of a UnixFS stuct. NOTE: this also calculates a sha256 hash + * @param data the data to write + * @param data_length the length of the data + * @param unix_fs the struct to add to + * @returns true(1) on success + */ +int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs) { + + unix_fs->bytes_size = data_length; + unix_fs->bytes = malloc(sizeof(unsigned char) * data_length); + if ( unix_fs->bytes == NULL) { + return 0; + } + memcpy( unix_fs->bytes, data, data_length); + + // now compute the hash + unix_fs->hash_length = 32; + unix_fs->hash = (unsigned char*)malloc(unix_fs->hash_length); + if (unix_fs->hash == NULL) { + free(unix_fs->bytes); + return 0; + } + if (libp2p_crypto_hashing_sha256(data, data_length, &unix_fs->hash[0]) == 0) { + free(unix_fs->bytes); + free(unix_fs->hash); + return 0; + } + + return 1; + +} + + /** * Protobuf functions */ @@ -68,7 +105,7 @@ enum WireType ipfs_unixfs_message_fields[] = { WIRETYPE_VARINT, WIRETYPE_LENGTH_ * @param obj what will be encoded * @returns the size of the buffer necessary to encode the object */ -size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj) { +size_t ipfs_unixfs_protobuf_encode_size(const struct UnixFS* obj) { size_t sz = 0; // bytes sz += obj->bytes_size + 11; @@ -111,8 +148,8 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou *bytes_written += bytes_used; } // file size (optional) - if (incoming->file_size > 0) { - retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used); + if (incoming->data_type == UNIXFS_FILE && incoming->bytes_size > 0) { + retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used); if (retVal == 0) return 0; *bytes_written += bytes_used; @@ -168,7 +205,7 @@ int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, s pos += bytes_read; break; case (3): // file size - result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read); + result->bytes_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read); pos += bytes_read; break; case (4): { // block sizes (linked list from varint)