Intermidiate commit with big changes to storage formats

I am attempting to match the storage format of the reference
implementation, so as to generate the same hashes.
This commit is contained in:
jmjatlanta 2016-12-23 09:37:43 -05:00
parent a569159cc2
commit 914d3caaed
19 changed files with 437 additions and 85 deletions

View file

@ -16,6 +16,7 @@ all:
cd flatfs; make all; cd flatfs; make all;
cd datastore; make all; cd datastore; make all;
cd thirdparty; make all; cd thirdparty; make all;
cd unixfs; make all;
cd main; make all; cd main; make all;
cd test; make all; cd test; make all;
@ -33,6 +34,7 @@ clean:
cd flatfs; make clean; cd flatfs; make clean;
cd datastore; make clean; cd datastore; make clean;
cd thirdparty; make clean; cd thirdparty; make clean;
cd unixfs; make clean;
cd main; make clean; cd main; make clean;
cd test; make clean; cd test; make clean;

View file

@ -40,12 +40,12 @@ unsigned char* ipfs_blockstore_cid_to_base32(const struct Cid* cid) {
return buffer; return buffer;
} }
unsigned char* ipfs_blockstore_hash_to_base32(const unsigned char* hash, size_t hash_size) { unsigned char* ipfs_blockstore_hash_to_base32(const unsigned char* hash, size_t hash_length) {
size_t key_length = libp2p_crypto_encoding_base32_encode_size(hash_size); size_t key_length = libp2p_crypto_encoding_base32_encode_size(hash_length);
unsigned char* buffer = (unsigned char*)malloc(key_length + 1); unsigned char* buffer = (unsigned char*)malloc(key_length + 1);
if (buffer == NULL) if (buffer == NULL)
return NULL; return NULL;
int retVal = ipfs_datastore_helper_ds_key_from_binary(hash, hash_size, &buffer[0], key_length, &key_length); int retVal = ipfs_datastore_helper_ds_key_from_binary(hash, hash_length, &buffer[0], key_length, &key_length);
if (retVal == 0) { if (retVal == 0) {
free(buffer); free(buffer);
return NULL; return NULL;
@ -145,3 +145,135 @@ int ipfs_blockstore_put(struct Block* block, struct FSRepo* fs_repo) {
free(filename); free(filename);
return 1; return 1;
} }
/***
* Put a struct UnixFS in the blockstore
* @param unix_fs the structure
* @param fs_repo the repo to place the strucure in
* @param bytes_written the number of bytes written to the blockstore
* @returns true(1) on success
*/
int ipfs_blockstore_put_unixfs(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written) {
// from blockstore.go line 118
int retVal = 0;
// Get Datastore key, which is a base32 key of the multihash,
unsigned char* key = ipfs_blockstore_hash_to_base32(unix_fs->hash, unix_fs->hash_length);
if (key == NULL) {
free(key);
return 0;
}
//TODO: put this in subdirectories
// turn the block into a binary array
size_t protobuf_len = ipfs_unixfs_protobuf_encode_size(unix_fs);
unsigned char protobuf[protobuf_len];
retVal = ipfs_unixfs_protobuf_encode(unix_fs, protobuf, protobuf_len, &protobuf_len);
if (retVal == 0) {
free(key);
return 0;
}
// now write byte array to file
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
if (filename == NULL) {
free(key);
return 0;
}
FILE* file = fopen(filename, "wb");
*bytes_written = fwrite(protobuf, 1, protobuf_len, file);
fclose(file);
if (*bytes_written != protobuf_len) {
free(key);
free(filename);
return 0;
}
// send to Put with key (this is now done separately)
//fs_repo->config->datastore->datastore_put(key, key_length, block->data, block->data_length, fs_repo->config->datastore);
free(key);
free(filename);
return 1;
}
/***
* Find a UnixFS struct based on its hash
* @param hash the hash to look for
* @param hash_length the length of the hash
* @param unix_fs the struct to fill
* @param fs_repo where to look for the data
* @returns true(1) on success
*/
int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, struct UnixFS** block, const struct FSRepo* fs_repo) {
// get datastore key, which is a base32 key of the multihash
unsigned char* key = ipfs_blockstore_hash_to_base32(hash, hash_length);
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
size_t file_size = os_utils_file_size(filename);
unsigned char buffer[file_size];
FILE* file = fopen(filename, "rb");
size_t bytes_read = fread(buffer, 1, file_size, file);
fclose(file);
int retVal = ipfs_unixfs_protobuf_decode(buffer, bytes_read, block);
free(key);
free(filename);
return retVal;
}
/***
* Put a struct Node in the blockstore
* @param node the structure
* @param fs_repo the repo to place the strucure in
* @param bytes_written the number of bytes written to the blockstore
* @returns true(1) on success
*/
int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written) {
// from blockstore.go line 118
int retVal = 0;
// Get Datastore key, which is a base32 key of the multihash,
unsigned char* key = ipfs_blockstore_hash_to_base32(node->hash, node->hash_size);
if (key == NULL) {
free(key);
return 0;
}
//TODO: put this in subdirectories
// turn the block into a binary array
size_t protobuf_len = ipfs_node_protobuf_encode_size(node);
unsigned char protobuf[protobuf_len];
retVal = ipfs_node_protobuf_encode(node, protobuf, protobuf_len, &protobuf_len);
if (retVal == 0) {
free(key);
return 0;
}
// now write byte array to file
char* filename = ipfs_blockstore_path_get(fs_repo, (char*)key);
if (filename == NULL) {
free(key);
return 0;
}
FILE* file = fopen(filename, "wb");
*bytes_written = fwrite(protobuf, 1, protobuf_len, file);
fclose(file);
if (*bytes_written != protobuf_len) {
free(key);
free(filename);
return 0;
}
free(key);
free(filename);
return 1;
}

View file

@ -23,27 +23,43 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
unsigned char buffer[MAX_DATA_SIZE]; unsigned char buffer[MAX_DATA_SIZE];
size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file); size_t bytes_read = fread(buffer, 1, MAX_DATA_SIZE, file);
// put the file bits into a new UnixFS file
struct UnixFS* new_unixfs = NULL;
ipfs_unixfs_new(&new_unixfs);
new_unixfs->data_type = UNIXFS_FILE;
ipfs_unixfs_add_data(&buffer[0], bytes_read, new_unixfs);
// protobuf the UnixFS
size_t protobuf_size = ipfs_unixfs_protobuf_encode_size(new_unixfs);
unsigned char protobuf[protobuf_size];
size_t bytes_written = 0;
ipfs_unixfs_protobuf_encode(new_unixfs, protobuf, protobuf_size, &bytes_written);
// we're done with the object
ipfs_unixfs_free(new_unixfs);
// create a new node // create a new node
struct Node* new_node = NULL; struct Node* new_node = NULL;
ipfs_node_new_from_data(buffer, bytes_read, &new_node); ipfs_node_new_from_data(protobuf, bytes_written, &new_node);
ipfs_node_set_hash(new_node, new_unixfs->hash, new_unixfs->hash_length);
// persist // persist
ipfs_merkledag_add(new_node, fs_repo); size_t size_of_node = 0;
ipfs_merkledag_add(new_node, fs_repo, &size_of_node);
// put link in parent node // put link in parent node
struct NodeLink* new_link = NULL; struct NodeLink* new_link = NULL;
ipfs_node_link_create("", new_node->hash, new_node->hash_size, &new_link); ipfs_node_link_create("", new_node->hash, new_node->hash_size, &new_link);
new_link->t_size = new_node->data_size; new_link->t_size = size_of_node;
*total_size += new_link->t_size; *total_size += new_link->t_size;
ipfs_node_add_link(parent_node, new_link); ipfs_node_add_link(parent_node, new_link);
ipfs_node_free(new_node); ipfs_node_free(new_node);
// save the parent_node if it is time...
if (bytes_read != MAX_DATA_SIZE) { if (bytes_read != MAX_DATA_SIZE) {
// build UnixFS for file // We have read everything, now save the parent_node,
// which is a sort of "directory" entry
/*
// build UnixFS for the parent
struct UnixFS* unix_fs; struct UnixFS* unix_fs;
if (ipfs_unixfs_new(&unix_fs) == 0) { if (ipfs_unixfs_new(&unix_fs) == 0) {
return 0; return 0;
} }
unix_fs->data_type = UNIXFS_FILE; unix_fs->data_type = UNIXFS_FILE;
unix_fs->file_size = *total_size; unix_fs->bytes_size = *total_size;
// now encode unixfs and put in parent_node->data // now encode unixfs and put in parent_node->data
size_t temp_size = ipfs_unixfs_protobuf_encode_size(unix_fs); size_t temp_size = ipfs_unixfs_protobuf_encode_size(unix_fs);
unsigned char temp[temp_size]; unsigned char temp[temp_size];
@ -59,8 +75,10 @@ size_t ipfs_import_chunk(FILE* file, struct Node* parent_node, struct FSRepo* fs
return 0; return 0;
} }
memcpy(parent_node->data, temp, bytes_written); memcpy(parent_node->data, temp, bytes_written);
ipfs_unixfs_free(unix_fs);
*/
// persist the main node // persist the main node
ipfs_merkledag_add(parent_node, fs_repo); ipfs_merkledag_add(parent_node, fs_repo, &bytes_written);
} }
return bytes_read; return bytes_read;
} }

View file

@ -37,5 +37,28 @@ int ipfs_blockstore_get(const unsigned char* hash, size_t hash_length, struct Bl
*/ */
int ipfs_blockstore_put(struct Block* block, const struct FSRepo* fs_repo); int ipfs_blockstore_put(struct Block* block, const struct FSRepo* fs_repo);
/***
* Put a struct UnixFS in the blockstore
* @param unix_fs the structure
* @param fs_repo the repo to place the strucure in
* @param bytes_written the number of bytes written to the blockstore
* @returns true(1) on success
*/
int ipfs_blockstore_put_unixfs(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written);
/***
* Find a UnixFS struct based on its hash
* @param hash the hash to look for
* @param hash_length the length of the hash
* @param unix_fs the struct to fill
* @param fs_repo where to look for the data
* @returns true(1) on success
*/
int ipfs_blockstore_get_unixfs(const unsigned char* hash, size_t hash_length, struct UnixFS** block, const struct FSRepo* fs_repo);
/**
* Put a struct Node in the blockstore
*/
int ipfs_blockstore_put_node(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written);
#endif #endif

View file

@ -10,10 +10,11 @@
/*** /***
* Adds a node to the dagService and blockService * Adds a node to the dagService and blockService
* @param node the node to add * @param node the node to add
* @param cid the resultant cid that was added * @param fs_repo the repo to add to
* @param bytes_written the number of bytes written
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo); int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_written);
/*** /***
* Retrieves a node from the datastore based on the cid * Retrieves a node from the datastore based on the cid

View file

@ -74,7 +74,7 @@ int ipfs_node_link_free(struct NodeLink * node_link);
* @param link the link to examine * @param link the link to examine
* @returns the maximum size that should be needed * @returns the maximum size that should be needed
*/ */
size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link); size_t ipfs_node_link_protobuf_encode_size(const struct NodeLink* link);
/*** /***
* Encode a NodeLink into protobuf format * Encode a NodeLink into protobuf format
@ -84,7 +84,7 @@ size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link);
* @pram bytes_written the amount of the buffer used * @pram bytes_written the amount of the buffer used
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_node_link_protobuf_encode(struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written); int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
/**** /****
* Decode from a byte array into a NodeLink * Decode from a byte array into a NodeLink
@ -102,7 +102,7 @@ int ipfs_node_link_protobuf_decode(unsigned char* buffer, size_t buffer_length,
* @param node the node to examine * @param node the node to examine
* @returns the max size of an encoded stream of bytes, if it were encoded * @returns the max size of an encoded stream of bytes, if it were encoded
*/ */
size_t ipfs_node_protobuf_encode_size(struct Node* node); size_t ipfs_node_protobuf_encode_size(const struct Node* node);
/*** /***
* Encode a node into a protobuf byte stream * Encode a node into a protobuf byte stream
@ -112,7 +112,7 @@ size_t ipfs_node_protobuf_encode_size(struct Node* node);
* @param bytes_written how much of buffer was used * @param bytes_written how much of buffer was used
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written); int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written);
/*** /***
* Decode a stream of bytes into a Node structure * Decode a stream of bytes into a Node structure

View file

@ -5,8 +5,10 @@
#define fs_repo_h #define fs_repo_h
#include <stdio.h> #include <stdio.h>
#include "ipfs/repo/config/config.h"
#include "ipfs/repo/config/config.h"
#include "ipfs/unixfs/unixfs.h"
#include "ipfs/merkledag/node.h"
/** /**
* a structure to hold the repo info * a structure to hold the repo info
@ -73,4 +75,23 @@ int ipfs_repo_fsrepo_init(struct FSRepo* config);
int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo); int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_repo);
int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo); int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo);
/***
* Write a unixfs to the datastore and blockstore
* @param unix_fs the struct to write
* @param fs_repo the repo to write to
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_unixfs_write(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written);
int ipfs_repo_fsrepo_unixfs_read(const unsigned char* hash, size_t hash_length, struct UnixFS** unix_fs, const struct FSRepo* fs_repo);
/***
* Write a struct Node to the datastore and blockstore
* @param node the struct to write
* @param fs_repo the repo to write to
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_node_write(const struct Node* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written);
int ipfs_repo_fsrepo_node_read(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo);
#endif /* fs_repo_h */ #endif /* fs_repo_h */

View file

@ -1,4 +1,4 @@
enum UnixFSFormatType { RAW, FILE, DIRECTORY, METADATA, SYMLINK }; enum UnixFSFormatType { RAW, UNIXFS_FILE, DIRECTORY, METADATA, SYMLINK };
struct UnixFSData { struct UnixFSData {
enum UnixFSFormatType type; enum UnixFSFormatType type;

View file

@ -43,8 +43,10 @@ struct UnixFS {
enum UnixFSDataType data_type; enum UnixFSDataType data_type;
size_t bytes_size; // the size of the bytes array size_t bytes_size; // the size of the bytes array
unsigned char* bytes; // an array of bytes unsigned char* bytes; // an array of bytes
size_t file_size; // the file size // size_t file_size; // the file size - I mimick this one
struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes struct UnixFSBlockSizeNode* block_size_head; // a linked list of block sizes
unsigned char* hash; // not saved
size_t hash_length; // not saved
}; };
struct UnixFSMetaData { struct UnixFSMetaData {
@ -65,6 +67,15 @@ int ipfs_unixfs_new(struct UnixFS** obj);
*/ */
int ipfs_unixfs_free(struct UnixFS* obj); int ipfs_unixfs_free(struct UnixFS* obj);
/***
* Write data to data section of a UnixFS stuct. NOTE: this also calculates a sha256 hash
* @param data the data to write
* @param data_length the length of the data
* @param unix_fs the struct to add to
* @returns true(1) on success
*/
int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs);
/** /**
* Protobuf functions * Protobuf functions
*/ */
@ -74,7 +85,7 @@ int ipfs_unixfs_free(struct UnixFS* obj);
* @param obj what will be encoded * @param obj what will be encoded
* @returns the size of the buffer necessary to encode the object * @returns the size of the buffer necessary to encode the object
*/ */
size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj); size_t ipfs_unixfs_protobuf_encode_size(const struct UnixFS* obj);
/*** /***
* Encode a UnixFS object into protobuf format * Encode a UnixFS object into protobuf format

View file

@ -9,14 +9,14 @@ OBJS = main.o \
../commands/argument.o ../commands/command_option.o ../commands/command.o ../commands/cli/parse.o \ ../commands/argument.o ../commands/command_option.o ../commands/command.o ../commands/cli/parse.o \
../core/builder.o \ ../core/builder.o \
../datastore/ds_helper.o \ ../datastore/ds_helper.o \
../dnslink/dnslink.o \
../flatfs/flatfs.o \ ../flatfs/flatfs.o \
../importer/importer.o \ ../importer/importer.o \
../importer/exporter.o \ ../importer/exporter.o \
../dnslink/dnslink.o \
../path/path.o \ ../path/path.o \
../namesys/isdomain.o \
../merkledag/merkledag.o ../merkledag/node.o \ ../merkledag/merkledag.o ../merkledag/node.o \
../multibase/multibase.o \ ../multibase/multibase.o \
../namesys/isdomain.o \
../os/utils.o \ ../os/utils.o \
../repo/init.o \ ../repo/init.o \
../repo/fsrepo/fs_repo.o ../repo/fsrepo/jsmn.o ../repo/fsrepo/lmdb_datastore.o \ ../repo/fsrepo/fs_repo.o ../repo/fsrepo/jsmn.o ../repo/fsrepo/lmdb_datastore.o \

BIN
main/ipfs

Binary file not shown.

View file

@ -1,41 +1,46 @@
/** /**
* A basic storage building block of the IPFS system * A basic storage building block of the IPFS system
*/ */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "libp2p/crypto/sha256.h"
#include "ipfs/merkledag/merkledag.h" #include "ipfs/merkledag/merkledag.h"
#include "ipfs/unixfs/unixfs.h"
/*** /***
* Adds a node to the dagService and blockService * Adds a node to the dagService and blockService
* @param node the node to add * @param node the node to add
* @param cid the resultant cid that was added * @param fs_repo the repo to add to
* @param bytes_written the number of bytes written
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) { int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo, size_t* bytes_written) {
// taken from merkledag.go line 59 // taken from merkledag.go line 59
int retVal = 0;
struct Block* block = NULL; // compute the hash
size_t protobuf_size = ipfs_node_protobuf_encode_size(node);
unsigned char protobuf[protobuf_size];
size_t bytes_encoded;
retVal = ipfs_node_protobuf_encode(node, protobuf, protobuf_size, &bytes_encoded);
// protobuf the node node->hash_size = 32;
size_t protobuf_len = ipfs_node_protobuf_encode_size(node); node->hash = (unsigned char*)malloc(node->hash_size);
size_t bytes_written = 0; if (node->hash == NULL) {
unsigned char protobuf[protobuf_len]; return 0;
ipfs_node_protobuf_encode(node, protobuf, protobuf_len, &bytes_written); }
if (libp2p_crypto_hashing_sha256(protobuf, bytes_encoded, &node->hash[0]) == 0) {
// turn the node into a block free(node->hash);
ipfs_blocks_block_new(&block);
ipfs_blocks_block_add_data(protobuf, bytes_written, block);
// write to block store & datastore
int retVal = ipfs_repo_fsrepo_block_write(block, fs_repo);
if (retVal == 0) {
ipfs_blocks_block_free(block);
return 0; return 0;
} }
ipfs_node_set_hash(node, block->cid->hash, block->cid->hash_length); // write to block store & datastore
retVal = ipfs_repo_fsrepo_node_write(node, fs_repo, bytes_written);
if (block != NULL) if (retVal == 0) {
ipfs_blocks_block_free(block); return 0;
}
// TODO: call HasBlock (unsure why as yet) // TODO: call HasBlock (unsure why as yet)
return 1; return 1;
@ -51,7 +56,8 @@ int ipfs_merkledag_add(struct Node* node, struct FSRepo* fs_repo) {
*/ */
int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node** node, const struct FSRepo* fs_repo) { int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node** node, const struct FSRepo* fs_repo) {
int retVal = 1; int retVal = 1;
struct Block* block; //struct Block* block;
struct UnixFS* unix_fs;
size_t key_length = 100; size_t key_length = 100;
unsigned char key[key_length]; unsigned char key[key_length];
@ -61,27 +67,27 @@ int ipfs_merkledag_get(const unsigned char* hash, size_t hash_size, struct Node*
if (retVal == 0) if (retVal == 0)
return 0; return 0;
// we have the record from the db. Go get the block from the blockstore // we have the record from the db. Go get the UnixFS from the blockstore
retVal = ipfs_repo_fsrepo_block_read(hash, hash_size, &block, fs_repo); retVal = ipfs_repo_fsrepo_unixfs_read(hash, hash_size, &unix_fs, fs_repo);
if (retVal == 0) { if (retVal == 0) {
return 0; return 0;
} }
// now convert the block into a node // now convert the block into a node
if (ipfs_node_protobuf_decode(block->data, block->data_length, node) == 0) { if (ipfs_node_protobuf_decode(unix_fs->bytes, unix_fs->bytes_size, node) == 0) {
ipfs_blocks_block_free(block); ipfs_unixfs_free(unix_fs);
return 0; return 0;
} }
// set the cid on the node // set the cid on the node
if (ipfs_node_set_hash(*node, hash, hash_size) == 0) { if (ipfs_node_set_hash(*node, hash, hash_size) == 0) {
ipfs_blocks_block_free(block); ipfs_unixfs_free(unix_fs);
ipfs_node_free(*node); ipfs_node_free(*node);
return 0; return 0;
} }
// free resources // free resources
ipfs_blocks_block_free(block); ipfs_unixfs_free(unix_fs);
return 1; return 1;
} }

View file

@ -10,7 +10,7 @@
#include "ipfs/merkledag/node.h" #include "ipfs/merkledag/node.h"
// for protobuf Node (all fields optional) links (repeated node_link) data (optional bytes) // for protobuf Node (all fields optional) data (optional bytes) links (repeated node_link)
enum WireType ipfs_node_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED }; enum WireType ipfs_node_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED };
// for protobuf NodeLink (all fields optional) hash name tsize // for protobuf NodeLink (all fields optional) hash name tsize
enum WireType ipfs_node_link_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED, WIRETYPE_VARINT }; enum WireType ipfs_node_link_message_fields[] = { WIRETYPE_LENGTH_DELIMITED, WIRETYPE_LENGTH_DELIMITED, WIRETYPE_VARINT };
@ -84,7 +84,7 @@ int ipfs_node_link_free(struct NodeLink * node_link)
* @param link the link to examine * @param link the link to examine
* @returns the maximum length of the encoded NodeLink * @returns the maximum length of the encoded NodeLink
*/ */
size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link) { size_t ipfs_node_link_protobuf_encode_size(const struct NodeLink* link) {
if (link == NULL) if (link == NULL)
return 0; return 0;
// hash, name, tsize // hash, name, tsize
@ -109,7 +109,7 @@ size_t ipfs_node_link_protobuf_encode_size(struct NodeLink* link) {
* @param bytes_written the number of bytes written to buffer * @param bytes_written the number of bytes written to buffer
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_node_link_protobuf_encode(struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) { int ipfs_node_link_protobuf_encode(const struct NodeLink* link, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
// 3 fields, hash (length delimited), name (length delimited), tsize (varint) // 3 fields, hash (length delimited), name (length delimited), tsize (varint)
size_t bytes_used = 0; size_t bytes_used = 0;
int retVal = 0; int retVal = 0;
@ -200,7 +200,7 @@ exit:
/*** /***
* return an approximate size of the encoded node * return an approximate size of the encoded node
*/ */
size_t ipfs_node_protobuf_encode_size(struct Node* node) { size_t ipfs_node_protobuf_encode_size(const struct Node* node) {
size_t size = 0; size_t size = 0;
// links // links
struct NodeLink* current = node->head_link; struct NodeLink* current = node->head_link;
@ -223,10 +223,17 @@ size_t ipfs_node_protobuf_encode_size(struct Node* node) {
* @param bytes_written how much of buffer was used * @param bytes_written how much of buffer was used
* @returns true(1) on success * @returns true(1) on success
*/ */
int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) { int ipfs_node_protobuf_encode(const struct Node* node, unsigned char* buffer, size_t max_buffer_length, size_t* bytes_written) {
size_t bytes_used = 0; size_t bytes_used = 0;
*bytes_written = 0; *bytes_written = 0;
int retVal = 0; int retVal = 0;
// data
if (node->data_size > 0) {
retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
}
// links // links
struct NodeLink* current = node->head_link; struct NodeLink* current = node->head_link;
while (current != NULL) { while (current != NULL) {
@ -235,19 +242,12 @@ int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t m
retVal = ipfs_node_link_protobuf_encode(current, temp, temp_size, &bytes_used); retVal = ipfs_node_link_protobuf_encode(current, temp, temp_size, &bytes_used);
if (retVal == 0) if (retVal == 0)
return 0; return 0;
retVal = protobuf_encode_length_delimited(1, ipfs_node_message_fields[0], (char*)temp, bytes_used, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used); retVal = protobuf_encode_length_delimited(2, ipfs_node_message_fields[1], (char*)temp, bytes_used, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
if (retVal == 0) if (retVal == 0)
return 0; return 0;
*bytes_written += bytes_used; *bytes_written += bytes_used;
current = current->next; current = current->next;
} }
// data
if (node->data_size > 0) {
retVal = protobuf_encode_length_delimited(2, ipfs_node_message_fields[1], (char*)node->data, node->data_size, &buffer[*bytes_written], max_buffer_length - *bytes_written, &bytes_used);
if (retVal == 0)
return 0;
*bytes_written += bytes_used;
}
return 1; return 1;
} }
@ -261,8 +261,8 @@ int ipfs_node_protobuf_encode(struct Node* node, unsigned char* buffer, size_t m
*/ */
int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struct Node** node) { int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struct Node** node) {
/* /*
* Field 0: link
* Field 1: data * Field 1: data
* Field 2: link
*/ */
size_t pos = 0; size_t pos = 0;
int retVal = 0; int retVal = 0;
@ -282,7 +282,13 @@ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struc
} }
pos += bytes_read; pos += bytes_read;
switch(field_no) { switch(field_no) {
case (1): {// links case (1): { // data
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*node)->data), &((*node)->data_size), &bytes_read) == 0)
goto exit;
pos += bytes_read;
break;
}
case (2): {// links
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0) if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&temp_buffer, &temp_size, &bytes_read) == 0)
goto exit; goto exit;
pos += bytes_read; pos += bytes_read;
@ -293,12 +299,6 @@ int ipfs_node_protobuf_decode(unsigned char* buffer, size_t buffer_length, struc
ipfs_node_add_link(*node, temp_link); ipfs_node_add_link(*node, temp_link);
break; break;
} }
case (2): { // data
if (protobuf_decode_length_delimited(&buffer[pos], buffer_length - pos, (char**)&((*node)->data), &((*node)->data_size), &bytes_read) == 0)
goto exit;
pos += bytes_read;
break;
}
} }
} }

View file

@ -554,6 +554,69 @@ int ipfs_repo_fsrepo_block_write(struct Block* block, const struct FSRepo* fs_re
return 1; return 1;
} }
/***
* Write a unixfs to the datastore and blockstore
* @param unix_fs the struct to write
* @param fs_repo the repo to write to
* @param bytes_written number of bytes written to the repo
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_unixfs_write(const struct UnixFS* unix_fs, const struct FSRepo* fs_repo, size_t* bytes_written) {
/**
* What is put in the blockstore is the block.
* What is put in the datastore is the multihash (the Cid) as the key,
* and the base32 encoded multihash as the value.
*/
int retVal = 1;
retVal = ipfs_blockstore_put_unixfs(unix_fs, fs_repo, bytes_written);
if (retVal == 0)
return 0;
// take the hash, base32 it, and send both to the datastore
size_t fs_key_length = 100;
unsigned char fs_key[fs_key_length];
retVal = ipfs_datastore_helper_ds_key_from_binary(unix_fs->hash, unix_fs->hash_length, fs_key, fs_key_length, &fs_key_length);
if (retVal == 0)
return 0;
retVal = fs_repo->config->datastore->datastore_put(unix_fs->hash, unix_fs->hash_length, fs_key, fs_key_length, fs_repo->config->datastore);
if (retVal == 0)
return 0;
return 1;
}
/***
* Write a unixfs to the datastore and blockstore
* @param unix_fs the struct to write
* @param fs_repo the repo to write to
* @param bytes_written number of bytes written to the repo
* @returns true(1) on success
*/
int ipfs_repo_fsrepo_node_write(const struct Node* node, const struct FSRepo* fs_repo, size_t* bytes_written) {
/**
* What is put in the blockstore is the node.
* What is put in the datastore is the multihash as the key,
* and the base32 encoded multihash as the value.
*/
int retVal = 1;
retVal = ipfs_blockstore_put_node(node, fs_repo, bytes_written);
if (retVal == 0)
return 0;
// take the hash, base32 it, and send both to the datastore
size_t fs_key_length = 100;
unsigned char fs_key[fs_key_length];
retVal = ipfs_datastore_helper_ds_key_from_binary(node->hash, node->hash_size, fs_key, fs_key_length, &fs_key_length);
if (retVal == 0)
return 0;
retVal = fs_repo->config->datastore->datastore_put(node->hash, node->hash_size, fs_key, fs_key_length, fs_repo->config->datastore);
if (retVal == 0)
return 0;
return 1;
}
int ipfs_repo_fsrepo_node_read(const unsigned char* hash, size_t hash_length, struct Node** node, const struct FSRepo* fs_repo) {
return 0;
}
int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo) { int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, struct Block** block, const struct FSRepo* fs_repo) {
int retVal = 0; int retVal = 0;
@ -569,3 +632,18 @@ int ipfs_repo_fsrepo_block_read(const unsigned char* hash, size_t hash_length, s
return retVal; return retVal;
} }
int ipfs_repo_fsrepo_unixfs_read(const unsigned char* hash, size_t hash_length, struct UnixFS** unix_fs, const struct FSRepo* fs_repo) {
int retVal = 0;
// get the base32 hash from the database
// We do this only to see if it is in the database
size_t fs_key_length = 100;
unsigned char fs_key[fs_key_length];
retVal = fs_repo->config->datastore->datastore_get((const char*)hash, hash_length, fs_key, fs_key_length, &fs_key_length, fs_repo->config->datastore);
if (retVal == 0) // maybe it doesn't exist?
return 0;
// now get the block from the blockstore
retVal = ipfs_blockstore_get_unixfs(hash, hash_length, unix_fs, fs_repo);
return retVal;
}

View file

@ -42,8 +42,8 @@ int test_merkledag_get_data() {
// create a node // create a node
struct Node* node1; struct Node* node1;
retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1); retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1);
size_t bytes_written = 0;
retVal = ipfs_merkledag_add(node1, fs_repo); retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
if (retVal == 0) { if (retVal == 0) {
ipfs_node_free(node1); ipfs_node_free(node1);
ipfs_repo_fsrepo_free(fs_repo); ipfs_repo_fsrepo_free(fs_repo);
@ -105,7 +105,8 @@ int test_merkledag_add_data() {
struct Node* node1; struct Node* node1;
retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1); retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node1);
retVal = ipfs_merkledag_add(node1, fs_repo); size_t bytes_written = 0;
retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
if (retVal == 0) { if (retVal == 0) {
ipfs_node_free(node1); ipfs_node_free(node1);
return 0; return 0;
@ -124,7 +125,7 @@ int test_merkledag_add_data() {
// adding the same binary again should do nothing (the hash should be the same) // adding the same binary again should do nothing (the hash should be the same)
struct Node* node2; struct Node* node2;
retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node2); retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node2);
retVal = ipfs_merkledag_add(node2, fs_repo); retVal = ipfs_merkledag_add(node2, fs_repo, &bytes_written);
if (retVal == 0) { if (retVal == 0) {
ipfs_node_free(node1); ipfs_node_free(node1);
ipfs_node_free(node2); ipfs_node_free(node2);
@ -160,7 +161,7 @@ int test_merkledag_add_data() {
struct Node* node3; struct Node* node3;
retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node3); retVal = ipfs_node_new_from_data(binary_data, binary_data_size, &node3);
retVal = ipfs_merkledag_add(node3, fs_repo); retVal = ipfs_merkledag_add(node3, fs_repo, &bytes_written);
if (retVal == 0) { if (retVal == 0) {
ipfs_node_free(node1); ipfs_node_free(node1);
ipfs_node_free(node2); ipfs_node_free(node2);
@ -207,7 +208,8 @@ int test_merkledag_add_node() {
return 0; return 0;
} }
retVal = ipfs_merkledag_add(node1, fs_repo); size_t bytes_written = 0;
retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
if (retVal == 0) { if (retVal == 0) {
ipfs_repo_fsrepo_free(fs_repo); ipfs_repo_fsrepo_free(fs_repo);
ipfs_node_free(node1); ipfs_node_free(node1);
@ -250,7 +252,8 @@ int test_merkledag_add_node_with_links() {
return 0; return 0;
} }
retVal = ipfs_merkledag_add(node1, fs_repo); size_t bytes_written = 0;
retVal = ipfs_merkledag_add(node1, fs_repo, &bytes_written);
if (retVal == 0) { if (retVal == 0) {
ipfs_repo_fsrepo_free(fs_repo); ipfs_repo_fsrepo_free(fs_repo);
ipfs_node_free(node1); ipfs_node_free(node1);

View file

@ -65,7 +65,6 @@ int test_unixfs_encode_smallfile() {
memcpy(unixfs->bytes, bytes, 35); memcpy(unixfs->bytes, bytes, 35);
unixfs->bytes_size = 35; unixfs->bytes_size = 35;
unixfs->data_type = UNIXFS_FILE; unixfs->data_type = UNIXFS_FILE;
unixfs->file_size = 35;
size_t protobuf_size = 43; size_t protobuf_size = 43;
unsigned char protobuf[protobuf_size]; unsigned char protobuf[protobuf_size];

18
unixfs/Makefile Normal file
View file

@ -0,0 +1,18 @@
CC = gcc
CFLAGS = -O0 -I../include -I../../c-libp2p/include -I../../c-multihash/include -I../../c-multiaddr/include -I../../c-protobuf -Wall
ifdef DEBUG
CFLAGS += -g3
endif
LFLAGS =
DEPS =
OBJS = unixfs.o
%.o: %.c $(DEPS)
$(CC) -c -o $@ $< $(CFLAGS)
all: $(OBJS)
clean:
rm -f *.o

View file

@ -1,7 +1,10 @@
#include <string.h> #include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include "ipfs/unixfs/format.h" #include "ipfs/unixfs/format.h"
/** /**
* Copy data into a new struct * Copy data into a new struct
* @param type the type of data * @param type the type of data
@ -12,7 +15,7 @@
*/ */
int ipfs_unixfs_format_new(enum UnixFSFormatType type, const unsigned char* data, int ipfs_unixfs_format_new(enum UnixFSFormatType type, const unsigned char* data,
size_t data_length, struct UnixFSData** result) { size_t data_length, struct UnixFSData** result) {
struct UnixFSData* new_struct = malloc(sizeof(struct UnixFSData)); struct UnixFSData* new_struct = (struct UnixFSData*)malloc(sizeof(struct UnixFSData));
if (new_struct == NULL) if (new_struct == NULL)
return 0; return 0;

View file

@ -4,7 +4,9 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h>
#include "libp2p/crypto/sha256.h"
#include "ipfs/unixfs/unixfs.h" #include "ipfs/unixfs/unixfs.h"
#include "protobuf.h" #include "protobuf.h"
#include "varint.h" #include "varint.h"
@ -43,8 +45,9 @@ int ipfs_unixfs_new(struct UnixFS** obj) {
(*obj)->bytes = 0; (*obj)->bytes = 0;
(*obj)->bytes_size = 0; (*obj)->bytes_size = 0;
(*obj)->data_type = UNIXFS_RAW; (*obj)->data_type = UNIXFS_RAW;
(*obj)->file_size = 0;
(*obj)->block_size_head = NULL; (*obj)->block_size_head = NULL;
(*obj)->hash = NULL;
(*obj)->hash_length = 0;
return 1; return 1;
} }
@ -56,6 +59,40 @@ int ipfs_unixfs_free(struct UnixFS* obj) {
return 1; return 1;
} }
/***
* Write data to data section of a UnixFS stuct. NOTE: this also calculates a sha256 hash
* @param data the data to write
* @param data_length the length of the data
* @param unix_fs the struct to add to
* @returns true(1) on success
*/
int ipfs_unixfs_add_data(unsigned char* data, size_t data_length, struct UnixFS* unix_fs) {
unix_fs->bytes_size = data_length;
unix_fs->bytes = malloc(sizeof(unsigned char) * data_length);
if ( unix_fs->bytes == NULL) {
return 0;
}
memcpy( unix_fs->bytes, data, data_length);
// now compute the hash
unix_fs->hash_length = 32;
unix_fs->hash = (unsigned char*)malloc(unix_fs->hash_length);
if (unix_fs->hash == NULL) {
free(unix_fs->bytes);
return 0;
}
if (libp2p_crypto_hashing_sha256(data, data_length, &unix_fs->hash[0]) == 0) {
free(unix_fs->bytes);
free(unix_fs->hash);
return 0;
}
return 1;
}
/** /**
* Protobuf functions * Protobuf functions
*/ */
@ -68,7 +105,7 @@ enum WireType ipfs_unixfs_message_fields[] = { WIRETYPE_VARINT, WIRETYPE_LENGTH_
* @param obj what will be encoded * @param obj what will be encoded
* @returns the size of the buffer necessary to encode the object * @returns the size of the buffer necessary to encode the object
*/ */
size_t ipfs_unixfs_protobuf_encode_size(struct UnixFS* obj) { size_t ipfs_unixfs_protobuf_encode_size(const struct UnixFS* obj) {
size_t sz = 0; size_t sz = 0;
// bytes // bytes
sz += obj->bytes_size + 11; sz += obj->bytes_size + 11;
@ -111,8 +148,8 @@ int ipfs_unixfs_protobuf_encode(const struct UnixFS* incoming, unsigned char* ou
*bytes_written += bytes_used; *bytes_written += bytes_used;
} }
// file size (optional) // file size (optional)
if (incoming->file_size > 0) { if (incoming->data_type == UNIXFS_FILE && incoming->bytes_size > 0) {
retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->file_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used); retVal = protobuf_encode_varint(3, ipfs_unixfs_message_fields[2], incoming->bytes_size, &outgoing[*bytes_written], max_buffer_size - (*bytes_written), &bytes_used);
if (retVal == 0) if (retVal == 0)
return 0; return 0;
*bytes_written += bytes_used; *bytes_written += bytes_used;
@ -168,7 +205,7 @@ int ipfs_unixfs_protobuf_decode(unsigned char* incoming, size_t incoming_size, s
pos += bytes_read; pos += bytes_read;
break; break;
case (3): // file size case (3): // file size
result->file_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read); result->bytes_size = varint_decode(&incoming[pos], incoming_size - pos, &bytes_read);
pos += bytes_read; pos += bytes_read;
break; break;
case (4): { // block sizes (linked list from varint) case (4): { // block sizes (linked list from varint)